@hraness/oh 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -21
- package/dist/cli.d.ts +1 -1
- package/dist/cli.js +1 -1
- package/dist/libsql-semantic-v2.d.ts +160 -0
- package/dist/libsql-semantic-v2.d.ts.map +1 -0
- package/dist/libsql-semantic.d.ts +19 -0
- package/dist/libsql-semantic.d.ts.map +1 -1
- package/dist/semantic-cloud.d.ts +1 -0
- package/dist/semantic-cloud.d.ts.map +1 -1
- package/dist/semantic-cloud.js +1570 -0
- package/package.json +1 -1
- package/skills/oh/SKILL.md +2 -2
- package/spec/README.md +8 -1
- package/spec/manifest.json +5 -1
- package/spec/v1/libsql-semantic-cache-schema-v1.sql +114 -0
- package/spec/v1/libsql-semantic-digest-fixture-v1.json +13 -0
- package/spec/v1/semantic-cloud.md +9 -0
- package/spec/v2/libsql-semantic-cache-schema-v2.sql +175 -0
- package/spec/v2/libsql-semantic-digest-fixture-v2.json +14 -0
- package/spec/v2/manifest.json +11 -0
- package/spec/v2/semantic-cloud.md +106 -0
- package/src/cli.ts +1 -1
- package/src/libsql-semantic-v2.test.ts +1114 -0
- package/src/libsql-semantic-v2.ts +1891 -0
- package/src/libsql-semantic.test.ts +157 -1
- package/src/libsql-semantic.ts +51 -0
- package/src/semantic-cloud.ts +1 -0
package/dist/semantic-cloud.js
CHANGED
|
@@ -1456,6 +1456,33 @@ class OhLibSqlSemanticCacheV1 {
|
|
|
1456
1456
|
if (this.#closeClient)
|
|
1457
1457
|
this.#client.close?.();
|
|
1458
1458
|
}
|
|
1459
|
+
async publishedHead(input) {
|
|
1460
|
+
this.#open();
|
|
1461
|
+
const authorityId = parseAuthorityId(input.authorityId);
|
|
1462
|
+
if (await readPurge(this.#client, authorityId) !== null)
|
|
1463
|
+
return null;
|
|
1464
|
+
const head = await readHead(this.#client, authorityId);
|
|
1465
|
+
if (head === null)
|
|
1466
|
+
return null;
|
|
1467
|
+
const generation = await readGeneration(this.#client, authorityId, head.generation);
|
|
1468
|
+
if (generation === null || !headMatchesGeneration(head, generation)) {
|
|
1469
|
+
if (await readPurge(this.#client, authorityId) !== null)
|
|
1470
|
+
return null;
|
|
1471
|
+
throw new OhLibSqlSemanticError("integrity", "The semantic published head does not match its immutable generation.");
|
|
1472
|
+
}
|
|
1473
|
+
if (await readPurge(this.#client, authorityId) !== null)
|
|
1474
|
+
return null;
|
|
1475
|
+
const finalHead = await readHead(this.#client, authorityId);
|
|
1476
|
+
if (finalHead === null) {
|
|
1477
|
+
if (await readPurge(this.#client, authorityId) !== null)
|
|
1478
|
+
return null;
|
|
1479
|
+
throw new OhLibSqlSemanticError("integrity", "The semantic published head disappeared during its read.");
|
|
1480
|
+
}
|
|
1481
|
+
if (canonicalJson(finalHead) !== canonicalJson(head)) {
|
|
1482
|
+
throw new OhLibSqlSemanticError("conflict", "The semantic published head changed during its read.");
|
|
1483
|
+
}
|
|
1484
|
+
return Object.freeze({ ...head, v: 1 });
|
|
1485
|
+
}
|
|
1459
1486
|
async stage(input) {
|
|
1460
1487
|
this.#open();
|
|
1461
1488
|
validateEmbeddingClient(input.embeddingClient);
|
|
@@ -1827,16 +1854,1559 @@ class OhLibSqlSemanticCacheV1 {
|
|
|
1827
1854
|
async function openOhLibSqlSemanticCacheV1(client, options = {}) {
|
|
1828
1855
|
return await OhLibSqlSemanticCacheV1.open(client, options.closeClient ?? false);
|
|
1829
1856
|
}
|
|
1857
|
+
// src/libsql-semantic-v2.ts
|
|
1858
|
+
var OH_LIBSQL_SEMANTIC_LIMITS_V2 = Object.freeze({
|
|
1859
|
+
chunksPerDocument: 64,
|
|
1860
|
+
chunksPerGeneration: 4096,
|
|
1861
|
+
documentsPerGeneration: 512,
|
|
1862
|
+
embeddingBatch: 16,
|
|
1863
|
+
searchLimit: 100,
|
|
1864
|
+
searchPage: 128
|
|
1865
|
+
});
|
|
1866
|
+
|
|
1867
|
+
class OhLibSqlSemanticV2Error extends Error {
|
|
1868
|
+
code;
|
|
1869
|
+
constructor(code, message) {
|
|
1870
|
+
super(message);
|
|
1871
|
+
this.name = "OhLibSqlSemanticV2Error";
|
|
1872
|
+
this.code = code;
|
|
1873
|
+
}
|
|
1874
|
+
}
|
|
1875
|
+
var SCHEMA_NAME_V1 = "oh.libsql-semantic-cache.v1";
|
|
1876
|
+
var SCHEMA_VERSION_V1 = 1;
|
|
1877
|
+
var SCHEMA_NAME2 = "oh.libsql-semantic-cache.v2";
|
|
1878
|
+
var SCHEMA_VERSION2 = 2;
|
|
1879
|
+
var VECTOR_BYTES2 = OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.dimensions * 4;
|
|
1880
|
+
var DEFAULT_ISOLATION_KIND = "oh.semantic-authority-isolation.v2";
|
|
1881
|
+
var GENERATION_KIND = "oh.semantic-generation.v2";
|
|
1882
|
+
var MEMBERSHIP_KIND = "oh.semantic-membership.v2";
|
|
1883
|
+
var PURGE_MARKER_KIND = "oh.semantic-purge-marker.v2";
|
|
1884
|
+
var PURGE_RECEIPT_KIND = "oh.semantic-purge-receipt.v2";
|
|
1885
|
+
var TRANSITION_PAGE_SIZE = 32;
|
|
1886
|
+
var TRANSITION_TABLE_NAME = "oh_semantic_v1_purge_transition";
|
|
1887
|
+
var SCHEMA_TABLE2 = `CREATE TABLE IF NOT EXISTS oh_semantic_schemas (
|
|
1888
|
+
version INTEGER PRIMARY KEY,
|
|
1889
|
+
name TEXT NOT NULL UNIQUE,
|
|
1890
|
+
schema_sha256 TEXT NOT NULL,
|
|
1891
|
+
applied_at TEXT NOT NULL
|
|
1892
|
+
) STRICT`;
|
|
1893
|
+
var TRANSITION_TABLE = `CREATE TABLE oh_semantic_v1_purge_transition (
|
|
1894
|
+
authority_id TEXT PRIMARY KEY,
|
|
1895
|
+
purged_at TEXT NOT NULL
|
|
1896
|
+
) STRICT`;
|
|
1897
|
+
var SCHEMA_STATEMENTS_V1 = Object.freeze([
|
|
1898
|
+
`CREATE TABLE IF NOT EXISTS oh_semantic_vectors (
|
|
1899
|
+
profile_sha256 TEXT NOT NULL,
|
|
1900
|
+
renderer_sha256 TEXT NOT NULL,
|
|
1901
|
+
input_sha256 TEXT NOT NULL,
|
|
1902
|
+
vector_sha256 TEXT NOT NULL,
|
|
1903
|
+
vector BLOB NOT NULL,
|
|
1904
|
+
created_at TEXT NOT NULL,
|
|
1905
|
+
PRIMARY KEY(profile_sha256, renderer_sha256, input_sha256)
|
|
1906
|
+
) STRICT`,
|
|
1907
|
+
`CREATE TABLE IF NOT EXISTS oh_semantic_generations (
|
|
1908
|
+
authority_id TEXT NOT NULL,
|
|
1909
|
+
generation INTEGER NOT NULL CHECK(generation >= 0),
|
|
1910
|
+
authority_sha256 TEXT NOT NULL,
|
|
1911
|
+
profile_sha256 TEXT NOT NULL,
|
|
1912
|
+
renderer_sha256 TEXT NOT NULL,
|
|
1913
|
+
membership_sha256 TEXT NOT NULL,
|
|
1914
|
+
generation_sha256 TEXT NOT NULL UNIQUE,
|
|
1915
|
+
document_count INTEGER NOT NULL CHECK(document_count >= 0),
|
|
1916
|
+
chunk_count INTEGER NOT NULL CHECK(chunk_count >= 0),
|
|
1917
|
+
created_at TEXT NOT NULL,
|
|
1918
|
+
PRIMARY KEY(authority_id, generation)
|
|
1919
|
+
) STRICT`,
|
|
1920
|
+
`CREATE TABLE IF NOT EXISTS oh_semantic_memberships (
|
|
1921
|
+
authority_id TEXT NOT NULL,
|
|
1922
|
+
generation INTEGER NOT NULL CHECK(generation >= 0),
|
|
1923
|
+
generation_sha256 TEXT NOT NULL,
|
|
1924
|
+
record_key TEXT NOT NULL,
|
|
1925
|
+
record_sha256 TEXT NOT NULL,
|
|
1926
|
+
ordinal INTEGER NOT NULL CHECK(ordinal >= 0),
|
|
1927
|
+
input_sha256 TEXT NOT NULL,
|
|
1928
|
+
PRIMARY KEY(authority_id, generation, record_key, ordinal)
|
|
1929
|
+
) STRICT`,
|
|
1930
|
+
`CREATE TABLE IF NOT EXISTS oh_semantic_heads (
|
|
1931
|
+
authority_id TEXT PRIMARY KEY,
|
|
1932
|
+
generation INTEGER NOT NULL CHECK(generation >= 0),
|
|
1933
|
+
authority_sha256 TEXT NOT NULL,
|
|
1934
|
+
profile_sha256 TEXT NOT NULL,
|
|
1935
|
+
renderer_sha256 TEXT NOT NULL,
|
|
1936
|
+
membership_sha256 TEXT NOT NULL,
|
|
1937
|
+
generation_sha256 TEXT NOT NULL,
|
|
1938
|
+
published_at TEXT NOT NULL
|
|
1939
|
+
) STRICT`,
|
|
1940
|
+
`CREATE TABLE IF NOT EXISTS oh_semantic_purges (
|
|
1941
|
+
authority_id TEXT PRIMARY KEY,
|
|
1942
|
+
purged_at TEXT NOT NULL
|
|
1943
|
+
) STRICT`,
|
|
1944
|
+
`CREATE INDEX IF NOT EXISTS oh_semantic_memberships_generation
|
|
1945
|
+
ON oh_semantic_memberships(authority_id, generation, record_key, ordinal)`,
|
|
1946
|
+
`CREATE INDEX IF NOT EXISTS oh_semantic_memberships_input
|
|
1947
|
+
ON oh_semantic_memberships(input_sha256)`,
|
|
1948
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_vectors_no_update
|
|
1949
|
+
BEFORE UPDATE ON oh_semantic_vectors
|
|
1950
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic vectors are immutable'); END`,
|
|
1951
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_generations_no_update
|
|
1952
|
+
BEFORE UPDATE ON oh_semantic_generations
|
|
1953
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic generations are immutable'); END`,
|
|
1954
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_memberships_no_update
|
|
1955
|
+
BEFORE UPDATE ON oh_semantic_memberships
|
|
1956
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic memberships are immutable'); END`,
|
|
1957
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_generations_purge_guard
|
|
1958
|
+
BEFORE INSERT ON oh_semantic_generations
|
|
1959
|
+
WHEN EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = NEW.authority_id)
|
|
1960
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic authority was purged'); END`,
|
|
1961
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_memberships_purge_guard
|
|
1962
|
+
BEFORE INSERT ON oh_semantic_memberships
|
|
1963
|
+
WHEN EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = NEW.authority_id)
|
|
1964
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic authority was purged'); END`,
|
|
1965
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_memberships_published_guard
|
|
1966
|
+
BEFORE INSERT ON oh_semantic_memberships
|
|
1967
|
+
WHEN EXISTS (SELECT 1 FROM oh_semantic_heads
|
|
1968
|
+
WHERE authority_id = NEW.authority_id AND generation = NEW.generation)
|
|
1969
|
+
AND NOT EXISTS (SELECT 1 FROM oh_semantic_memberships
|
|
1970
|
+
WHERE authority_id = NEW.authority_id AND generation = NEW.generation
|
|
1971
|
+
AND generation_sha256 = NEW.generation_sha256
|
|
1972
|
+
AND record_key = NEW.record_key AND record_sha256 = NEW.record_sha256
|
|
1973
|
+
AND ordinal = NEW.ordinal AND input_sha256 = NEW.input_sha256)
|
|
1974
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic generation is published'); END`,
|
|
1975
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_heads_insert_purge_guard
|
|
1976
|
+
BEFORE INSERT ON oh_semantic_heads
|
|
1977
|
+
WHEN EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = NEW.authority_id)
|
|
1978
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic authority was purged'); END`,
|
|
1979
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_heads_update_purge_guard
|
|
1980
|
+
BEFORE UPDATE ON oh_semantic_heads
|
|
1981
|
+
WHEN EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = NEW.authority_id)
|
|
1982
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic authority was purged'); END`,
|
|
1983
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_purges_no_update
|
|
1984
|
+
BEFORE UPDATE ON oh_semantic_purges
|
|
1985
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic purge markers are immutable'); END`,
|
|
1986
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_purges_no_delete
|
|
1987
|
+
BEFORE DELETE ON oh_semantic_purges
|
|
1988
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic purge markers are immutable'); END`
|
|
1989
|
+
]);
|
|
1990
|
+
var SCHEMA_STATEMENTS2 = Object.freeze([
|
|
1991
|
+
`CREATE TABLE IF NOT EXISTS oh_semantic_isolations (
|
|
1992
|
+
isolation_sha256 TEXT PRIMARY KEY,
|
|
1993
|
+
authority_id TEXT NOT NULL,
|
|
1994
|
+
created_at TEXT NOT NULL
|
|
1995
|
+
) STRICT`,
|
|
1996
|
+
`CREATE TABLE IF NOT EXISTS oh_semantic_vectors (
|
|
1997
|
+
isolation_sha256 TEXT NOT NULL,
|
|
1998
|
+
profile_sha256 TEXT NOT NULL,
|
|
1999
|
+
renderer_sha256 TEXT NOT NULL,
|
|
2000
|
+
input_sha256 TEXT NOT NULL,
|
|
2001
|
+
vector_sha256 TEXT NOT NULL,
|
|
2002
|
+
vector BLOB NOT NULL,
|
|
2003
|
+
created_at TEXT NOT NULL,
|
|
2004
|
+
PRIMARY KEY(isolation_sha256, profile_sha256, renderer_sha256, input_sha256)
|
|
2005
|
+
) STRICT`,
|
|
2006
|
+
`CREATE TABLE IF NOT EXISTS oh_semantic_generations (
|
|
2007
|
+
authority_id TEXT NOT NULL,
|
|
2008
|
+
generation INTEGER NOT NULL CHECK(generation >= 0),
|
|
2009
|
+
authority_sha256 TEXT NOT NULL,
|
|
2010
|
+
isolation_sha256 TEXT NOT NULL,
|
|
2011
|
+
profile_sha256 TEXT NOT NULL,
|
|
2012
|
+
renderer_sha256 TEXT NOT NULL,
|
|
2013
|
+
membership_sha256 TEXT NOT NULL,
|
|
2014
|
+
generation_sha256 TEXT NOT NULL UNIQUE,
|
|
2015
|
+
document_count INTEGER NOT NULL CHECK(document_count >= 0),
|
|
2016
|
+
chunk_count INTEGER NOT NULL CHECK(chunk_count >= 0),
|
|
2017
|
+
created_at TEXT NOT NULL,
|
|
2018
|
+
PRIMARY KEY(authority_id, generation)
|
|
2019
|
+
) STRICT`,
|
|
2020
|
+
`CREATE TABLE IF NOT EXISTS oh_semantic_memberships (
|
|
2021
|
+
authority_id TEXT NOT NULL,
|
|
2022
|
+
generation INTEGER NOT NULL CHECK(generation >= 0),
|
|
2023
|
+
generation_sha256 TEXT NOT NULL,
|
|
2024
|
+
isolation_sha256 TEXT NOT NULL,
|
|
2025
|
+
record_key TEXT NOT NULL,
|
|
2026
|
+
record_sha256 TEXT NOT NULL,
|
|
2027
|
+
ordinal INTEGER NOT NULL CHECK(ordinal >= 0),
|
|
2028
|
+
input_sha256 TEXT NOT NULL,
|
|
2029
|
+
PRIMARY KEY(authority_id, generation, record_key, ordinal)
|
|
2030
|
+
) STRICT`,
|
|
2031
|
+
`CREATE TABLE IF NOT EXISTS oh_semantic_heads (
|
|
2032
|
+
authority_id TEXT PRIMARY KEY,
|
|
2033
|
+
generation INTEGER NOT NULL CHECK(generation >= 0),
|
|
2034
|
+
authority_sha256 TEXT NOT NULL,
|
|
2035
|
+
isolation_sha256 TEXT NOT NULL,
|
|
2036
|
+
profile_sha256 TEXT NOT NULL,
|
|
2037
|
+
renderer_sha256 TEXT NOT NULL,
|
|
2038
|
+
membership_sha256 TEXT NOT NULL,
|
|
2039
|
+
generation_sha256 TEXT NOT NULL,
|
|
2040
|
+
published_at TEXT NOT NULL
|
|
2041
|
+
) STRICT`,
|
|
2042
|
+
`CREATE TABLE IF NOT EXISTS oh_semantic_purges (
|
|
2043
|
+
authority_id TEXT PRIMARY KEY,
|
|
2044
|
+
isolation_sha256 TEXT NOT NULL,
|
|
2045
|
+
profile_sha256 TEXT NOT NULL,
|
|
2046
|
+
published_generation INTEGER CHECK(published_generation IS NULL OR published_generation >= 0),
|
|
2047
|
+
published_generation_sha256 TEXT,
|
|
2048
|
+
purged_at TEXT NOT NULL,
|
|
2049
|
+
purge_marker_sha256 TEXT NOT NULL,
|
|
2050
|
+
generation_count INTEGER NOT NULL CHECK(generation_count >= 0),
|
|
2051
|
+
membership_count INTEGER NOT NULL CHECK(membership_count >= 0),
|
|
2052
|
+
orphan_vector_count INTEGER NOT NULL CHECK(orphan_vector_count >= 0),
|
|
2053
|
+
isolation_scope_count INTEGER NOT NULL CHECK(isolation_scope_count >= 0),
|
|
2054
|
+
counts_recorded INTEGER NOT NULL CHECK(counts_recorded IN (0, 1))
|
|
2055
|
+
) STRICT`,
|
|
2056
|
+
`CREATE INDEX IF NOT EXISTS oh_semantic_isolations_authority
|
|
2057
|
+
ON oh_semantic_isolations(authority_id, isolation_sha256)`,
|
|
2058
|
+
`CREATE INDEX IF NOT EXISTS oh_semantic_memberships_generation
|
|
2059
|
+
ON oh_semantic_memberships(authority_id, generation, record_key, ordinal)`,
|
|
2060
|
+
`CREATE INDEX IF NOT EXISTS oh_semantic_memberships_input
|
|
2061
|
+
ON oh_semantic_memberships(isolation_sha256, input_sha256)`,
|
|
2062
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_isolations_no_update
|
|
2063
|
+
BEFORE UPDATE ON oh_semantic_isolations
|
|
2064
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic isolations are immutable'); END`,
|
|
2065
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_isolations_no_delete
|
|
2066
|
+
BEFORE DELETE ON oh_semantic_isolations
|
|
2067
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic isolations are immutable'); END`,
|
|
2068
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_isolations_purge_guard
|
|
2069
|
+
BEFORE INSERT ON oh_semantic_isolations
|
|
2070
|
+
WHEN EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = NEW.authority_id)
|
|
2071
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic authority was purged'); END`,
|
|
2072
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_vectors_no_update
|
|
2073
|
+
BEFORE UPDATE ON oh_semantic_vectors
|
|
2074
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic vectors are immutable'); END`,
|
|
2075
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_vectors_isolation_guard
|
|
2076
|
+
BEFORE INSERT ON oh_semantic_vectors
|
|
2077
|
+
WHEN NOT EXISTS (SELECT 1 FROM oh_semantic_isolations
|
|
2078
|
+
WHERE isolation_sha256 = NEW.isolation_sha256)
|
|
2079
|
+
OR EXISTS (SELECT 1 FROM oh_semantic_purges AS purge
|
|
2080
|
+
JOIN oh_semantic_isolations AS isolation
|
|
2081
|
+
ON isolation.authority_id = purge.authority_id
|
|
2082
|
+
WHERE isolation.isolation_sha256 = NEW.isolation_sha256)
|
|
2083
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic vector isolation is unavailable'); END`,
|
|
2084
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_generations_no_update
|
|
2085
|
+
BEFORE UPDATE ON oh_semantic_generations
|
|
2086
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic generations are immutable'); END`,
|
|
2087
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_memberships_no_update
|
|
2088
|
+
BEFORE UPDATE ON oh_semantic_memberships
|
|
2089
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic memberships are immutable'); END`,
|
|
2090
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_generations_purge_guard
|
|
2091
|
+
BEFORE INSERT ON oh_semantic_generations
|
|
2092
|
+
WHEN EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = NEW.authority_id)
|
|
2093
|
+
OR NOT EXISTS (SELECT 1 FROM oh_semantic_isolations
|
|
2094
|
+
WHERE isolation_sha256 = NEW.isolation_sha256 AND authority_id = NEW.authority_id)
|
|
2095
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic authority or isolation is unavailable'); END`,
|
|
2096
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_memberships_purge_guard
|
|
2097
|
+
BEFORE INSERT ON oh_semantic_memberships
|
|
2098
|
+
WHEN EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = NEW.authority_id)
|
|
2099
|
+
OR NOT EXISTS (SELECT 1 FROM oh_semantic_generations
|
|
2100
|
+
WHERE authority_id = NEW.authority_id AND generation = NEW.generation
|
|
2101
|
+
AND generation_sha256 = NEW.generation_sha256
|
|
2102
|
+
AND isolation_sha256 = NEW.isolation_sha256)
|
|
2103
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic authority or isolation is unavailable'); END`,
|
|
2104
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_memberships_published_guard
|
|
2105
|
+
BEFORE INSERT ON oh_semantic_memberships
|
|
2106
|
+
WHEN EXISTS (SELECT 1 FROM oh_semantic_heads
|
|
2107
|
+
WHERE authority_id = NEW.authority_id AND generation = NEW.generation)
|
|
2108
|
+
AND NOT EXISTS (SELECT 1 FROM oh_semantic_memberships
|
|
2109
|
+
WHERE authority_id = NEW.authority_id AND generation = NEW.generation
|
|
2110
|
+
AND generation_sha256 = NEW.generation_sha256
|
|
2111
|
+
AND isolation_sha256 = NEW.isolation_sha256
|
|
2112
|
+
AND record_key = NEW.record_key AND record_sha256 = NEW.record_sha256
|
|
2113
|
+
AND ordinal = NEW.ordinal AND input_sha256 = NEW.input_sha256)
|
|
2114
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic generation is published'); END`,
|
|
2115
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_heads_insert_purge_guard
|
|
2116
|
+
BEFORE INSERT ON oh_semantic_heads
|
|
2117
|
+
WHEN EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = NEW.authority_id)
|
|
2118
|
+
OR NOT EXISTS (SELECT 1 FROM oh_semantic_generations
|
|
2119
|
+
WHERE authority_id = NEW.authority_id AND generation = NEW.generation
|
|
2120
|
+
AND generation_sha256 = NEW.generation_sha256
|
|
2121
|
+
AND isolation_sha256 = NEW.isolation_sha256)
|
|
2122
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic authority or isolation is unavailable'); END`,
|
|
2123
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_heads_update_purge_guard
|
|
2124
|
+
BEFORE UPDATE ON oh_semantic_heads
|
|
2125
|
+
WHEN EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = NEW.authority_id)
|
|
2126
|
+
OR NOT EXISTS (SELECT 1 FROM oh_semantic_generations
|
|
2127
|
+
WHERE authority_id = NEW.authority_id AND generation = NEW.generation
|
|
2128
|
+
AND generation_sha256 = NEW.generation_sha256
|
|
2129
|
+
AND isolation_sha256 = NEW.isolation_sha256)
|
|
2130
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic authority or isolation is unavailable'); END`,
|
|
2131
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_purges_no_update
|
|
2132
|
+
BEFORE UPDATE ON oh_semantic_purges
|
|
2133
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic purge markers are immutable'); END`,
|
|
2134
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_purges_no_delete
|
|
2135
|
+
BEFORE DELETE ON oh_semantic_purges
|
|
2136
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic purge markers are immutable'); END`
|
|
2137
|
+
]);
|
|
2138
|
+
function normalizedSchemaSql2(sql) {
|
|
2139
|
+
return sql.replace(/\bIF\s+NOT\s+EXISTS\b/giu, "").replace(/\s+/gu, " ").trim();
|
|
2140
|
+
}
|
|
2141
|
+
function expectedSchemaObject2(statement) {
|
|
2142
|
+
const match = /^CREATE\s+(TABLE|INDEX|TRIGGER)(?:\s+IF\s+NOT\s+EXISTS)?\s+([a-z0-9_]+)/iu.exec(statement.trim());
|
|
2143
|
+
if (match === null)
|
|
2144
|
+
throw new Error("Invalid compiled semantic schema statement.");
|
|
2145
|
+
const declared = match[1]?.toLowerCase();
|
|
2146
|
+
const type = declared === "index" ? "index" : declared === "trigger" ? "trigger" : "table";
|
|
2147
|
+
const name = match[2];
|
|
2148
|
+
const owner = type === "table" ? name : /\bON\s+([a-z0-9_]+)/iu.exec(statement)?.[1];
|
|
2149
|
+
if (owner === undefined)
|
|
2150
|
+
throw new Error("Invalid compiled semantic schema owner.");
|
|
2151
|
+
return { name, sql: normalizedSchemaSql2(statement), tableName: owner, type };
|
|
2152
|
+
}
|
|
2153
|
+
var EXPECTED_SCHEMA_OBJECTS2 = Object.freeze([SCHEMA_TABLE2, ...SCHEMA_STATEMENTS2].map(expectedSchemaObject2).sort((left, right) => canonicalJson([left.type, left.name]).localeCompare(canonicalJson([right.type, right.name]))));
|
|
2154
|
+
var SCHEMA_SHA2562 = canonicalSha256(EXPECTED_SCHEMA_OBJECTS2);
|
|
2155
|
+
var EXPECTED_SCHEMA_OBJECTS_V1 = Object.freeze([SCHEMA_TABLE2, ...SCHEMA_STATEMENTS_V1].map(expectedSchemaObject2).sort((left, right) => canonicalJson([left.type, left.name]).localeCompare(canonicalJson([right.type, right.name]))));
|
|
2156
|
+
var SCHEMA_SHA256_V1 = canonicalSha256(EXPECTED_SCHEMA_OBJECTS_V1);
|
|
2157
|
+
var EXPECTED_TRANSITION_SCHEMA_OBJECTS = Object.freeze([...EXPECTED_SCHEMA_OBJECTS2, expectedSchemaObject2(TRANSITION_TABLE)].sort((left, right) => canonicalJson([left.type, left.name]).localeCompare(canonicalJson([right.type, right.name]))));
|
|
2158
|
+
function rowValue2(row, key, index) {
|
|
2159
|
+
return Array.isArray(row) ? row[index] : row[key];
|
|
2160
|
+
}
|
|
2161
|
+
function integer2(value) {
|
|
2162
|
+
if (typeof value === "number")
|
|
2163
|
+
return Number.isSafeInteger(value) ? value : null;
|
|
2164
|
+
if (typeof value === "bigint") {
|
|
2165
|
+
const converted = Number(value);
|
|
2166
|
+
return Number.isSafeInteger(converted) ? converted : null;
|
|
2167
|
+
}
|
|
2168
|
+
return null;
|
|
2169
|
+
}
|
|
2170
|
+
function rowsAffected2(result) {
|
|
2171
|
+
return typeof result.rowsAffected === "number" && Number.isSafeInteger(result.rowsAffected) && result.rowsAffected >= 0 ? result.rowsAffected : 0;
|
|
2172
|
+
}
|
|
2173
|
+
function parseAuthorityId2(value) {
|
|
2174
|
+
const parsed = safeCode(value, 256);
|
|
2175
|
+
if (parsed === null)
|
|
2176
|
+
throw new OhLibSqlSemanticV2Error("invalid-input", "Invalid semantic authority ID.");
|
|
2177
|
+
return parsed;
|
|
2178
|
+
}
|
|
2179
|
+
function deriveOhSemanticIsolationSha256V2(authorityId) {
|
|
2180
|
+
return canonicalSha256({
|
|
2181
|
+
authorityId: parseAuthorityId2(authorityId),
|
|
2182
|
+
kind: DEFAULT_ISOLATION_KIND,
|
|
2183
|
+
v: 2
|
|
2184
|
+
});
|
|
2185
|
+
}
|
|
2186
|
+
function isolationSha256(authorityId, value) {
|
|
2187
|
+
return value === undefined ? deriveOhSemanticIsolationSha256V2(authorityId) : parseDigest2(value, "semantic isolation");
|
|
2188
|
+
}
|
|
2189
|
+
function purgeMarkerSha256(authorityId, isolation, purgedAt) {
|
|
2190
|
+
return canonicalSha256({
|
|
2191
|
+
authorityId,
|
|
2192
|
+
isolationSha256: isolation,
|
|
2193
|
+
kind: PURGE_MARKER_KIND,
|
|
2194
|
+
purgedAt,
|
|
2195
|
+
v: 2
|
|
2196
|
+
});
|
|
2197
|
+
}
|
|
2198
|
+
function parseRecordKey2(value) {
|
|
2199
|
+
const parsed = safeCode(value, 512);
|
|
2200
|
+
if (parsed === null)
|
|
2201
|
+
throw new OhLibSqlSemanticV2Error("invalid-input", "Invalid semantic record key.");
|
|
2202
|
+
return parsed;
|
|
2203
|
+
}
|
|
2204
|
+
function parseGeneration2(value) {
|
|
2205
|
+
if (!Number.isSafeInteger(value) || value < 0) {
|
|
2206
|
+
throw new OhLibSqlSemanticV2Error("invalid-input", "Invalid semantic authority generation.");
|
|
2207
|
+
}
|
|
2208
|
+
return value;
|
|
2209
|
+
}
|
|
2210
|
+
function parseDigest2(value, label) {
|
|
2211
|
+
const digest = parseSha256Hex(value);
|
|
2212
|
+
if (digest === null)
|
|
2213
|
+
throw new OhLibSqlSemanticV2Error("invalid-input", `Invalid ${label} digest.`);
|
|
2214
|
+
return digest;
|
|
2215
|
+
}
|
|
2216
|
+
function parseInstant2(value) {
|
|
2217
|
+
const instant = parseCanonicalInstantV1(value);
|
|
2218
|
+
if (instant === null)
|
|
2219
|
+
throw new OhLibSqlSemanticV2Error("invalid-input", "Invalid semantic instant.");
|
|
2220
|
+
return instant;
|
|
2221
|
+
}
|
|
2222
|
+
async function schemaObjects2(client) {
|
|
2223
|
+
const result = await client.execute(`SELECT type, name, tbl_name, sql FROM sqlite_schema
|
|
2224
|
+
WHERE sql IS NOT NULL AND (name GLOB 'oh_semantic_*' OR tbl_name GLOB 'oh_semantic_*')
|
|
2225
|
+
ORDER BY type, name`);
|
|
2226
|
+
return result.rows.map((row) => {
|
|
2227
|
+
const type = rowValue2(row, "type", 0);
|
|
2228
|
+
const name = rowValue2(row, "name", 1);
|
|
2229
|
+
const tableName = rowValue2(row, "tbl_name", 2);
|
|
2230
|
+
const sql = rowValue2(row, "sql", 3);
|
|
2231
|
+
if (type !== "index" && type !== "table" && type !== "trigger" || typeof name !== "string" || typeof tableName !== "string" || typeof sql !== "string") {
|
|
2232
|
+
throw new OhLibSqlSemanticV2Error("integrity", "The semantic schema inventory is malformed.");
|
|
2233
|
+
}
|
|
2234
|
+
const schemaType = type;
|
|
2235
|
+
return { name, sql: normalizedSchemaSql2(sql), tableName, type: schemaType };
|
|
2236
|
+
}).sort((left, right) => canonicalJson([left.type, left.name]).localeCompare(canonicalJson([right.type, right.name])));
|
|
2237
|
+
}
|
|
2238
|
+
async function verifySchemaRevision(client, revision) {
|
|
2239
|
+
let marker;
|
|
2240
|
+
try {
|
|
2241
|
+
marker = await client.execute({
|
|
2242
|
+
args: [revision.version],
|
|
2243
|
+
sql: "SELECT name, schema_sha256 FROM oh_semantic_schemas WHERE version = ?"
|
|
2244
|
+
});
|
|
2245
|
+
} catch {
|
|
2246
|
+
throw new OhLibSqlSemanticV2Error("schema-unavailable", "The semantic cache schema is unavailable.");
|
|
2247
|
+
}
|
|
2248
|
+
const row = marker.rows[0];
|
|
2249
|
+
if (marker.rows.length !== 1 || row === undefined || rowValue2(row, "name", 0) !== revision.name || rowValue2(row, "schema_sha256", 1) !== revision.schemaSha256) {
|
|
2250
|
+
throw new OhLibSqlSemanticV2Error("schema-unavailable", "The semantic cache schema marker is invalid.");
|
|
2251
|
+
}
|
|
2252
|
+
if (canonicalJson(await schemaObjects2(client)) !== canonicalJson(revision.expected)) {
|
|
2253
|
+
throw new OhLibSqlSemanticV2Error("integrity", "The semantic cache schema has drifted.");
|
|
2254
|
+
}
|
|
2255
|
+
}
|
|
2256
|
+
async function verifySchema2(client) {
|
|
2257
|
+
const transition = await client.execute({
|
|
2258
|
+
args: [TRANSITION_TABLE_NAME],
|
|
2259
|
+
sql: "SELECT name FROM sqlite_schema WHERE type = 'table' AND name = ?"
|
|
2260
|
+
});
|
|
2261
|
+
if (transition.rows.length !== 0) {
|
|
2262
|
+
throw new OhLibSqlSemanticV2Error("schema-unavailable", "The semantic V2 cache upgrade is still materializing purge custody.");
|
|
2263
|
+
}
|
|
2264
|
+
await verifySchemaRevision(client, {
|
|
2265
|
+
expected: EXPECTED_SCHEMA_OBJECTS2,
|
|
2266
|
+
name: SCHEMA_NAME2,
|
|
2267
|
+
schemaSha256: SCHEMA_SHA2562,
|
|
2268
|
+
version: SCHEMA_VERSION2
|
|
2269
|
+
});
|
|
2270
|
+
}
|
|
2271
|
+
async function transitionPage(client) {
|
|
2272
|
+
const result = await client.execute({
|
|
2273
|
+
args: [TRANSITION_PAGE_SIZE],
|
|
2274
|
+
sql: `SELECT authority_id, purged_at FROM oh_semantic_v1_purge_transition
|
|
2275
|
+
ORDER BY authority_id LIMIT ?`
|
|
2276
|
+
});
|
|
2277
|
+
return Object.freeze(result.rows.map((row) => {
|
|
2278
|
+
const authorityId = safeCode(rowValue2(row, "authority_id", 0), 256);
|
|
2279
|
+
const purgedAt = parseCanonicalInstantV1(rowValue2(row, "purged_at", 1));
|
|
2280
|
+
if (authorityId === null || purgedAt === null) {
|
|
2281
|
+
throw new OhLibSqlSemanticV2Error("integrity", "A semantic v1 purge marker is invalid.");
|
|
2282
|
+
}
|
|
2283
|
+
return Object.freeze({ authorityId, purgedAt });
|
|
2284
|
+
}));
|
|
2285
|
+
}
|
|
2286
|
+
async function verifyTransitionSchema(client) {
|
|
2287
|
+
if (canonicalJson(await schemaObjects2(client)) !== canonicalJson(EXPECTED_TRANSITION_SCHEMA_OBJECTS)) {
|
|
2288
|
+
throw new OhLibSqlSemanticV2Error("integrity", "The semantic V2 transition schema has drifted.");
|
|
2289
|
+
}
|
|
2290
|
+
const markers = await client.execute("SELECT version FROM oh_semantic_schemas ORDER BY version LIMIT 1");
|
|
2291
|
+
if (markers.rows.length !== 0) {
|
|
2292
|
+
throw new OhLibSqlSemanticV2Error("integrity", "The semantic V2 marker cannot exist during purge transition.");
|
|
2293
|
+
}
|
|
2294
|
+
}
|
|
2295
|
+
async function startSemanticCacheV1ToV2Transition(client) {
|
|
2296
|
+
try {
|
|
2297
|
+
await verifySchemaRevision(client, {
|
|
2298
|
+
expected: EXPECTED_SCHEMA_OBJECTS_V1,
|
|
2299
|
+
name: SCHEMA_NAME_V1,
|
|
2300
|
+
schemaSha256: SCHEMA_SHA256_V1,
|
|
2301
|
+
version: SCHEMA_VERSION_V1
|
|
2302
|
+
});
|
|
2303
|
+
await client.batch([
|
|
2304
|
+
{ sql: TRANSITION_TABLE },
|
|
2305
|
+
{
|
|
2306
|
+
sql: `INSERT INTO oh_semantic_v1_purge_transition(authority_id, purged_at)
|
|
2307
|
+
SELECT authority_id, purged_at FROM oh_semantic_purges`
|
|
2308
|
+
},
|
|
2309
|
+
{ sql: "DROP TABLE oh_semantic_heads" },
|
|
2310
|
+
{ sql: "DROP TABLE oh_semantic_memberships" },
|
|
2311
|
+
{ sql: "DROP TABLE oh_semantic_generations" },
|
|
2312
|
+
{ sql: "DROP TABLE oh_semantic_vectors" },
|
|
2313
|
+
{ sql: "DROP TABLE oh_semantic_purges" },
|
|
2314
|
+
{ sql: "DROP TABLE oh_semantic_schemas" },
|
|
2315
|
+
{ sql: SCHEMA_TABLE2 },
|
|
2316
|
+
...SCHEMA_STATEMENTS2.map((sql) => ({ sql }))
|
|
2317
|
+
], "write");
|
|
2318
|
+
} catch {
|
|
2319
|
+
const inventory = await schemaObjects2(client);
|
|
2320
|
+
if (canonicalJson(inventory) === canonicalJson(EXPECTED_SCHEMA_OBJECTS2)) {
|
|
2321
|
+
await verifySchema2(client);
|
|
2322
|
+
return;
|
|
2323
|
+
}
|
|
2324
|
+
if (canonicalJson(inventory) !== canonicalJson(EXPECTED_TRANSITION_SCHEMA_OBJECTS)) {
|
|
2325
|
+
throw new OhLibSqlSemanticV2Error("integrity", "The semantic V1-to-V2 transition did not begin atomically.");
|
|
2326
|
+
}
|
|
2327
|
+
}
|
|
2328
|
+
await verifyTransitionSchema(client);
|
|
2329
|
+
}
|
|
2330
|
+
async function materializeTransitionPage(client, purges) {
|
|
2331
|
+
const statements = [];
|
|
2332
|
+
for (const purge of purges) {
|
|
2333
|
+
const isolation = deriveOhSemanticIsolationSha256V2(purge.authorityId);
|
|
2334
|
+
const marker = purgeMarkerSha256(purge.authorityId, isolation, purge.purgedAt);
|
|
2335
|
+
statements.push({
|
|
2336
|
+
args: [
|
|
2337
|
+
isolation,
|
|
2338
|
+
purge.authorityId,
|
|
2339
|
+
purge.purgedAt,
|
|
2340
|
+
isolation,
|
|
2341
|
+
purge.authorityId,
|
|
2342
|
+
purge.purgedAt
|
|
2343
|
+
],
|
|
2344
|
+
sql: `INSERT INTO oh_semantic_isolations(isolation_sha256, authority_id, created_at)
|
|
2345
|
+
SELECT ?, ?, ? WHERE NOT EXISTS (SELECT 1 FROM oh_semantic_isolations
|
|
2346
|
+
WHERE isolation_sha256 = ? AND authority_id = ? AND created_at = ?)
|
|
2347
|
+
ON CONFLICT DO NOTHING`
|
|
2348
|
+
});
|
|
2349
|
+
statements.push({
|
|
2350
|
+
args: [
|
|
2351
|
+
purge.authorityId,
|
|
2352
|
+
isolation,
|
|
2353
|
+
OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256,
|
|
2354
|
+
purge.purgedAt,
|
|
2355
|
+
marker
|
|
2356
|
+
],
|
|
2357
|
+
sql: `INSERT INTO oh_semantic_purges(authority_id, isolation_sha256,
|
|
2358
|
+
profile_sha256, published_generation, published_generation_sha256,
|
|
2359
|
+
purged_at, purge_marker_sha256, generation_count, membership_count,
|
|
2360
|
+
orphan_vector_count, isolation_scope_count, counts_recorded)
|
|
2361
|
+
VALUES (?, ?, ?, NULL, NULL, ?, ?, 0, 0, 0, 1, 0)
|
|
2362
|
+
ON CONFLICT DO NOTHING`
|
|
2363
|
+
});
|
|
2364
|
+
statements.push({
|
|
2365
|
+
args: [
|
|
2366
|
+
purge.authorityId,
|
|
2367
|
+
purge.purgedAt,
|
|
2368
|
+
isolation,
|
|
2369
|
+
purge.authorityId,
|
|
2370
|
+
purge.purgedAt,
|
|
2371
|
+
purge.authorityId,
|
|
2372
|
+
isolation,
|
|
2373
|
+
OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256,
|
|
2374
|
+
purge.purgedAt,
|
|
2375
|
+
marker
|
|
2376
|
+
],
|
|
2377
|
+
sql: `DELETE FROM oh_semantic_v1_purge_transition
|
|
2378
|
+
WHERE authority_id = ? AND purged_at = ?
|
|
2379
|
+
AND EXISTS (SELECT 1 FROM oh_semantic_isolations
|
|
2380
|
+
WHERE isolation_sha256 = ? AND authority_id = ? AND created_at = ?)
|
|
2381
|
+
AND EXISTS (SELECT 1 FROM oh_semantic_purges
|
|
2382
|
+
WHERE authority_id = ? AND isolation_sha256 = ? AND profile_sha256 = ?
|
|
2383
|
+
AND published_generation IS NULL AND published_generation_sha256 IS NULL
|
|
2384
|
+
AND purged_at = ? AND purge_marker_sha256 = ?
|
|
2385
|
+
AND generation_count = 0 AND membership_count = 0
|
|
2386
|
+
AND orphan_vector_count = 0 AND isolation_scope_count = 1
|
|
2387
|
+
AND counts_recorded = 0)`
|
|
2388
|
+
});
|
|
2389
|
+
}
|
|
2390
|
+
await client.batch(statements, "write");
|
|
2391
|
+
const placeholders = purges.map(() => "?").join(", ");
|
|
2392
|
+
const remaining = await client.execute({
|
|
2393
|
+
args: purges.map(({ authorityId }) => authorityId),
|
|
2394
|
+
sql: `SELECT authority_id FROM oh_semantic_v1_purge_transition
|
|
2395
|
+
WHERE authority_id IN (${placeholders}) LIMIT 1`
|
|
2396
|
+
});
|
|
2397
|
+
if (remaining.rows.length !== 0) {
|
|
2398
|
+
throw new OhLibSqlSemanticV2Error("integrity", "A semantic V1 purge tombstone did not materialize exactly in V2.");
|
|
2399
|
+
}
|
|
2400
|
+
}
|
|
2401
|
+
async function finishSemanticCacheV2Transition(client, appliedAt) {
|
|
2402
|
+
try {
|
|
2403
|
+
await client.batch([
|
|
2404
|
+
{
|
|
2405
|
+
args: [SCHEMA_VERSION2, SCHEMA_NAME2, SCHEMA_SHA2562, appliedAt],
|
|
2406
|
+
sql: `INSERT INTO oh_semantic_schemas(version, name, schema_sha256, applied_at)
|
|
2407
|
+
SELECT ?, CASE WHEN NOT EXISTS
|
|
2408
|
+
(SELECT 1 FROM oh_semantic_v1_purge_transition) THEN ? END, ?, ?`
|
|
2409
|
+
},
|
|
2410
|
+
{ sql: "DROP TABLE oh_semantic_v1_purge_transition" }
|
|
2411
|
+
], "write");
|
|
2412
|
+
} catch {
|
|
2413
|
+
await verifySchema2(client);
|
|
2414
|
+
}
|
|
2415
|
+
}
|
|
2416
|
+
async function hasConvergedSemanticCacheV2(client) {
|
|
2417
|
+
try {
|
|
2418
|
+
await verifySchema2(client);
|
|
2419
|
+
return true;
|
|
2420
|
+
} catch {
|
|
2421
|
+
return false;
|
|
2422
|
+
}
|
|
2423
|
+
}
|
|
2424
|
+
async function resumeSemanticCacheV2Transition(client, appliedAt) {
|
|
2425
|
+
try {
|
|
2426
|
+
await verifyTransitionSchema(client);
|
|
2427
|
+
for (;; ) {
|
|
2428
|
+
const page = await transitionPage(client);
|
|
2429
|
+
if (page.length === 0)
|
|
2430
|
+
break;
|
|
2431
|
+
await materializeTransitionPage(client, page);
|
|
2432
|
+
}
|
|
2433
|
+
await finishSemanticCacheV2Transition(client, appliedAt);
|
|
2434
|
+
} catch (error) {
|
|
2435
|
+
if (await hasConvergedSemanticCacheV2(client))
|
|
2436
|
+
return;
|
|
2437
|
+
throw error;
|
|
2438
|
+
}
|
|
2439
|
+
}
|
|
2440
|
+
async function bootstrapOhLibSqlSemanticCacheV2(client, options = {}) {
|
|
2441
|
+
const appliedAt = parseInstant2(options.appliedAt ?? canonicalNow());
|
|
2442
|
+
let existing = await schemaObjects2(client);
|
|
2443
|
+
if (existing.length === 0) {
|
|
2444
|
+
await client.batch([
|
|
2445
|
+
{ sql: SCHEMA_TABLE2 },
|
|
2446
|
+
...SCHEMA_STATEMENTS2.map((sql) => ({ sql })),
|
|
2447
|
+
{
|
|
2448
|
+
args: [SCHEMA_VERSION2, SCHEMA_NAME2, SCHEMA_SHA2562, appliedAt],
|
|
2449
|
+
sql: `INSERT INTO oh_semantic_schemas(version, name, schema_sha256, applied_at)
|
|
2450
|
+
VALUES (?, ?, ?, ?) ON CONFLICT(version) DO NOTHING`
|
|
2451
|
+
}
|
|
2452
|
+
], "write");
|
|
2453
|
+
} else if (canonicalJson(existing) === canonicalJson(EXPECTED_SCHEMA_OBJECTS_V1)) {
|
|
2454
|
+
await startSemanticCacheV1ToV2Transition(client);
|
|
2455
|
+
}
|
|
2456
|
+
existing = await schemaObjects2(client);
|
|
2457
|
+
if (canonicalJson(existing) === canonicalJson(EXPECTED_TRANSITION_SCHEMA_OBJECTS)) {
|
|
2458
|
+
await resumeSemanticCacheV2Transition(client, appliedAt);
|
|
2459
|
+
} else if (canonicalJson(existing) !== canonicalJson(EXPECTED_SCHEMA_OBJECTS2)) {
|
|
2460
|
+
throw new OhLibSqlSemanticV2Error("integrity", "Refusing to bless a partial or drifted semantic V2 schema.");
|
|
2461
|
+
}
|
|
2462
|
+
await verifySchema2(client);
|
|
2463
|
+
return Object.freeze({ schemaSha256: SCHEMA_SHA2562, schemaVersion: 2, v: 2 });
|
|
2464
|
+
}
|
|
2465
|
+
function vectorBytes2(vector) {
|
|
2466
|
+
const normalized = normalizeOhEmbeddingV1(vector);
|
|
2467
|
+
const bytes = new Uint8Array(VECTOR_BYTES2);
|
|
2468
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
2469
|
+
for (const [index, component] of normalized.entries())
|
|
2470
|
+
view.setFloat32(index * 4, component, true);
|
|
2471
|
+
return bytes;
|
|
2472
|
+
}
|
|
2473
|
+
function storedBytes2(value) {
|
|
2474
|
+
if (value instanceof Uint8Array)
|
|
2475
|
+
return new Uint8Array(value);
|
|
2476
|
+
if (value instanceof ArrayBuffer)
|
|
2477
|
+
return new Uint8Array(value.slice(0));
|
|
2478
|
+
return null;
|
|
2479
|
+
}
|
|
2480
|
+
function decodeVector2(value, expectedSha256) {
|
|
2481
|
+
const bytes = storedBytes2(value);
|
|
2482
|
+
const digest = parseSha256Hex(expectedSha256);
|
|
2483
|
+
if (bytes === null || bytes.byteLength !== VECTOR_BYTES2 || digest === null || sha256Hex(bytes) !== digest) {
|
|
2484
|
+
throw new OhLibSqlSemanticV2Error("integrity", "A cached semantic vector is corrupt.");
|
|
2485
|
+
}
|
|
2486
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
2487
|
+
const vector = Array.from({ length: OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.dimensions }, (_, index) => view.getFloat32(index * 4, true));
|
|
2488
|
+
try {
|
|
2489
|
+
return Object.freeze([...normalizeOhEmbeddingV1(vector)]);
|
|
2490
|
+
} catch {
|
|
2491
|
+
throw new OhLibSqlSemanticV2Error("integrity", "A cached semantic vector is invalid.");
|
|
2492
|
+
}
|
|
2493
|
+
}
|
|
2494
|
+
function prepareGeneration2(input) {
|
|
2495
|
+
const authorityId = parseAuthorityId2(input.authorityId);
|
|
2496
|
+
const authoritySha256 = parseDigest2(input.authoritySha256, "authority");
|
|
2497
|
+
const isolation = isolationSha256(authorityId, input.isolationSha256);
|
|
2498
|
+
const generation = parseGeneration2(input.generation);
|
|
2499
|
+
const createdAt = parseInstant2(input.createdAt ?? canonicalNow());
|
|
2500
|
+
const maximumChunks = input.maximumChunksPerDocument ?? OH_LIBSQL_SEMANTIC_LIMITS_V2.chunksPerDocument;
|
|
2501
|
+
if (!Number.isSafeInteger(maximumChunks) || maximumChunks < 1 || maximumChunks > OH_LIBSQL_SEMANTIC_LIMITS_V2.chunksPerDocument || !Array.isArray(input.documents) || input.documents.length < 1 || input.documents.length > OH_LIBSQL_SEMANTIC_LIMITS_V2.documentsPerGeneration) {
|
|
2502
|
+
throw new OhLibSqlSemanticV2Error("invalid-input", "Invalid semantic generation bounds.");
|
|
2503
|
+
}
|
|
2504
|
+
const documents = input.documents.map((document) => {
|
|
2505
|
+
if (document.v !== 2)
|
|
2506
|
+
throw new OhLibSqlSemanticV2Error("invalid-input", "Invalid semantic document version.");
|
|
2507
|
+
return {
|
|
2508
|
+
...document,
|
|
2509
|
+
key: parseRecordKey2(document.key),
|
|
2510
|
+
recordSha256: parseDigest2(document.recordSha256, "record")
|
|
2511
|
+
};
|
|
2512
|
+
}).sort((left, right) => left.key < right.key ? -1 : left.key > right.key ? 1 : 0);
|
|
2513
|
+
if (new Set(documents.map(({ key }) => key)).size !== documents.length) {
|
|
2514
|
+
throw new OhLibSqlSemanticV2Error("invalid-input", "Semantic document keys must be unique.");
|
|
2515
|
+
}
|
|
2516
|
+
const memberships = [];
|
|
2517
|
+
for (const document of documents) {
|
|
2518
|
+
const rendered = renderOhCloudflareEmbeddingDocumentV1({
|
|
2519
|
+
content: document.content,
|
|
2520
|
+
maximumChunks,
|
|
2521
|
+
title: document.title
|
|
2522
|
+
});
|
|
2523
|
+
if (rendered.status !== "complete") {
|
|
2524
|
+
throw new OhLibSqlSemanticV2Error("invalid-input", "A semantic document exceeds the complete renderer bound.");
|
|
2525
|
+
}
|
|
2526
|
+
for (const chunk of rendered.chunks) {
|
|
2527
|
+
memberships.push(Object.freeze({
|
|
2528
|
+
input: chunk.input,
|
|
2529
|
+
inputSha256: chunk.input.inputSha256,
|
|
2530
|
+
ordinal: chunk.ordinal,
|
|
2531
|
+
recordKey: document.key,
|
|
2532
|
+
recordSha256: document.recordSha256
|
|
2533
|
+
}));
|
|
2534
|
+
}
|
|
2535
|
+
}
|
|
2536
|
+
if (memberships.length < 1 || memberships.length > OH_LIBSQL_SEMANTIC_LIMITS_V2.chunksPerGeneration) {
|
|
2537
|
+
throw new OhLibSqlSemanticV2Error("invalid-input", "The semantic generation exceeds its chunk bound.");
|
|
2538
|
+
}
|
|
2539
|
+
const membershipSha256 = canonicalSha256({
|
|
2540
|
+
kind: MEMBERSHIP_KIND,
|
|
2541
|
+
memberships: memberships.map((membership) => ({
|
|
2542
|
+
inputSha256: membership.inputSha256,
|
|
2543
|
+
isolationSha256: isolation,
|
|
2544
|
+
ordinal: membership.ordinal,
|
|
2545
|
+
recordKey: membership.recordKey,
|
|
2546
|
+
recordSha256: membership.recordSha256
|
|
2547
|
+
})),
|
|
2548
|
+
v: 2
|
|
2549
|
+
});
|
|
2550
|
+
const generationSha256 = canonicalSha256({
|
|
2551
|
+
authorityId,
|
|
2552
|
+
authoritySha256,
|
|
2553
|
+
chunkCount: memberships.length,
|
|
2554
|
+
documentCount: documents.length,
|
|
2555
|
+
generation,
|
|
2556
|
+
isolationSha256: isolation,
|
|
2557
|
+
kind: GENERATION_KIND,
|
|
2558
|
+
membershipSha256,
|
|
2559
|
+
profileSha256: OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256,
|
|
2560
|
+
rendererSha256: OH_SEMANTIC_RENDERER_V1.rendererSha256,
|
|
2561
|
+
v: 2
|
|
2562
|
+
});
|
|
2563
|
+
return Object.freeze({
|
|
2564
|
+
authorityId,
|
|
2565
|
+
authoritySha256,
|
|
2566
|
+
chunkCount: memberships.length,
|
|
2567
|
+
createdAt,
|
|
2568
|
+
documentCount: documents.length,
|
|
2569
|
+
generation,
|
|
2570
|
+
generationSha256,
|
|
2571
|
+
isolationSha256: isolation,
|
|
2572
|
+
membershipSha256,
|
|
2573
|
+
memberships: Object.freeze(memberships)
|
|
2574
|
+
});
|
|
2575
|
+
}
|
|
2576
|
+
function parseStoredGeneration2(row) {
|
|
2577
|
+
const authorityId = safeCode(rowValue2(row, "authority_id", 0), 256);
|
|
2578
|
+
const generation = integer2(rowValue2(row, "generation", 1));
|
|
2579
|
+
const authoritySha256 = parseSha256Hex(rowValue2(row, "authority_sha256", 2));
|
|
2580
|
+
const isolation = parseSha256Hex(rowValue2(row, "isolation_sha256", 3));
|
|
2581
|
+
const profileSha256 = parseSha256Hex(rowValue2(row, "profile_sha256", 4));
|
|
2582
|
+
const rendererSha256 = parseSha256Hex(rowValue2(row, "renderer_sha256", 5));
|
|
2583
|
+
const membershipSha256 = parseSha256Hex(rowValue2(row, "membership_sha256", 6));
|
|
2584
|
+
const generationSha256 = parseSha256Hex(rowValue2(row, "generation_sha256", 7));
|
|
2585
|
+
const documentCount = integer2(rowValue2(row, "document_count", 8));
|
|
2586
|
+
const chunkCount = integer2(rowValue2(row, "chunk_count", 9));
|
|
2587
|
+
const createdAtValue = rowValue2(row, "created_at", 10);
|
|
2588
|
+
const createdAt = parseCanonicalInstantV1(createdAtValue);
|
|
2589
|
+
if (authorityId === null || generation === null || generation < 0 || authoritySha256 === null || isolation === null || profileSha256 === null || rendererSha256 === null || membershipSha256 === null || generationSha256 === null || documentCount === null || documentCount < 1 || documentCount > OH_LIBSQL_SEMANTIC_LIMITS_V2.documentsPerGeneration || chunkCount === null || chunkCount < 1 || chunkCount > OH_LIBSQL_SEMANTIC_LIMITS_V2.chunksPerGeneration || createdAt === null) {
|
|
2590
|
+
throw new OhLibSqlSemanticV2Error("integrity", "A stored semantic generation is invalid.");
|
|
2591
|
+
}
|
|
2592
|
+
return Object.freeze({
|
|
2593
|
+
authorityId,
|
|
2594
|
+
authoritySha256,
|
|
2595
|
+
chunkCount,
|
|
2596
|
+
createdAt,
|
|
2597
|
+
documentCount,
|
|
2598
|
+
generation,
|
|
2599
|
+
generationSha256,
|
|
2600
|
+
isolationSha256: isolation,
|
|
2601
|
+
membershipSha256,
|
|
2602
|
+
profileSha256,
|
|
2603
|
+
rendererSha256
|
|
2604
|
+
});
|
|
2605
|
+
}
|
|
2606
|
+
function generationMatches2(left, right) {
|
|
2607
|
+
return left.authorityId === right.authorityId && left.authoritySha256 === right.authoritySha256 && left.chunkCount === right.chunkCount && left.documentCount === right.documentCount && left.generation === right.generation && left.generationSha256 === right.generationSha256 && left.isolationSha256 === right.isolationSha256 && left.membershipSha256 === right.membershipSha256 && left.profileSha256 === OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256 && left.rendererSha256 === OH_SEMANTIC_RENDERER_V1.rendererSha256;
|
|
2608
|
+
}
|
|
2609
|
+
function parseStoredHead2(row) {
|
|
2610
|
+
const authorityId = safeCode(rowValue2(row, "authority_id", 0), 256);
|
|
2611
|
+
const generation = integer2(rowValue2(row, "generation", 1));
|
|
2612
|
+
const authoritySha256 = parseSha256Hex(rowValue2(row, "authority_sha256", 2));
|
|
2613
|
+
const isolation = parseSha256Hex(rowValue2(row, "isolation_sha256", 3));
|
|
2614
|
+
const profileSha256 = parseSha256Hex(rowValue2(row, "profile_sha256", 4));
|
|
2615
|
+
const rendererSha256 = parseSha256Hex(rowValue2(row, "renderer_sha256", 5));
|
|
2616
|
+
const membershipSha256 = parseSha256Hex(rowValue2(row, "membership_sha256", 6));
|
|
2617
|
+
const generationSha256 = parseSha256Hex(rowValue2(row, "generation_sha256", 7));
|
|
2618
|
+
const publishedAtValue = rowValue2(row, "published_at", 8);
|
|
2619
|
+
const publishedAt = parseCanonicalInstantV1(publishedAtValue);
|
|
2620
|
+
if (authorityId === null || generation === null || generation < 0 || authoritySha256 === null || isolation === null || profileSha256 === null || rendererSha256 === null || membershipSha256 === null || generationSha256 === null || publishedAt === null) {
|
|
2621
|
+
throw new OhLibSqlSemanticV2Error("integrity", "A stored semantic head is invalid.");
|
|
2622
|
+
}
|
|
2623
|
+
return Object.freeze({
|
|
2624
|
+
authorityId,
|
|
2625
|
+
authoritySha256,
|
|
2626
|
+
generation,
|
|
2627
|
+
generationSha256,
|
|
2628
|
+
isolationSha256: isolation,
|
|
2629
|
+
membershipSha256,
|
|
2630
|
+
profileSha256,
|
|
2631
|
+
publishedAt,
|
|
2632
|
+
rendererSha256
|
|
2633
|
+
});
|
|
2634
|
+
}
|
|
2635
|
+
function headMatchesGeneration2(head, generation) {
|
|
2636
|
+
return head.authorityId === generation.authorityId && head.authoritySha256 === generation.authoritySha256 && head.generation === generation.generation && head.generationSha256 === generation.generationSha256 && head.isolationSha256 === generation.isolationSha256 && head.membershipSha256 === generation.membershipSha256 && head.profileSha256 === generation.profileSha256 && head.rendererSha256 === generation.rendererSha256;
|
|
2637
|
+
}
|
|
2638
|
+
var GENERATION_SELECT2 = `SELECT authority_id, generation, authority_sha256,
|
|
2639
|
+
isolation_sha256, profile_sha256, renderer_sha256, membership_sha256, generation_sha256,
|
|
2640
|
+
document_count, chunk_count, created_at
|
|
2641
|
+
FROM oh_semantic_generations WHERE authority_id = ? AND generation = ?`;
|
|
2642
|
+
var HEAD_SELECT2 = `SELECT authority_id, generation, authority_sha256,
|
|
2643
|
+
isolation_sha256, profile_sha256, renderer_sha256, membership_sha256,
|
|
2644
|
+
generation_sha256, published_at
|
|
2645
|
+
FROM oh_semantic_heads WHERE authority_id = ?`;
|
|
2646
|
+
async function readGeneration2(client, authorityId, generation) {
|
|
2647
|
+
const result = await client.execute({ args: [authorityId, generation], sql: GENERATION_SELECT2 });
|
|
2648
|
+
if (result.rows.length > 1)
|
|
2649
|
+
throw new OhLibSqlSemanticV2Error("integrity", "Duplicate semantic generations.");
|
|
2650
|
+
const row = result.rows[0];
|
|
2651
|
+
return row === undefined ? null : parseStoredGeneration2(row);
|
|
2652
|
+
}
|
|
2653
|
+
async function readHead2(client, authorityId) {
|
|
2654
|
+
const result = await client.execute({ args: [authorityId], sql: HEAD_SELECT2 });
|
|
2655
|
+
if (result.rows.length > 1)
|
|
2656
|
+
throw new OhLibSqlSemanticV2Error("integrity", "Duplicate semantic heads.");
|
|
2657
|
+
const row = result.rows[0];
|
|
2658
|
+
return row === undefined ? null : parseStoredHead2(row);
|
|
2659
|
+
}
|
|
2660
|
+
function purgeResult(stored) {
|
|
2661
|
+
return Object.freeze({
|
|
2662
|
+
...stored,
|
|
2663
|
+
purgeReceiptSha256: canonicalSha256({
|
|
2664
|
+
kind: PURGE_RECEIPT_KIND,
|
|
2665
|
+
receipt: stored,
|
|
2666
|
+
v: 2
|
|
2667
|
+
})
|
|
2668
|
+
});
|
|
2669
|
+
}
|
|
2670
|
+
async function readPurge2(client, authorityId) {
|
|
2671
|
+
const result = await client.execute({
|
|
2672
|
+
args: [authorityId],
|
|
2673
|
+
sql: `SELECT isolation_sha256, profile_sha256, published_generation,
|
|
2674
|
+
published_generation_sha256, purged_at, purge_marker_sha256,
|
|
2675
|
+
generation_count, membership_count, orphan_vector_count,
|
|
2676
|
+
isolation_scope_count, counts_recorded
|
|
2677
|
+
FROM oh_semantic_purges WHERE authority_id = ?`
|
|
2678
|
+
});
|
|
2679
|
+
if (result.rows.length > 1)
|
|
2680
|
+
throw new OhLibSqlSemanticV2Error("integrity", "Duplicate semantic purge markers.");
|
|
2681
|
+
const row = result.rows[0];
|
|
2682
|
+
if (row === undefined)
|
|
2683
|
+
return null;
|
|
2684
|
+
const isolation = parseSha256Hex(rowValue2(row, "isolation_sha256", 0));
|
|
2685
|
+
const profileSha256 = parseSha256Hex(rowValue2(row, "profile_sha256", 1));
|
|
2686
|
+
const publishedGenerationValue = rowValue2(row, "published_generation", 2);
|
|
2687
|
+
const publishedGeneration = publishedGenerationValue === null ? null : integer2(publishedGenerationValue);
|
|
2688
|
+
const publishedGenerationSha256Value = rowValue2(row, "published_generation_sha256", 3);
|
|
2689
|
+
const publishedGenerationSha256 = publishedGenerationSha256Value === null ? null : parseSha256Hex(publishedGenerationSha256Value);
|
|
2690
|
+
const purgedAt = parseCanonicalInstantV1(rowValue2(row, "purged_at", 4));
|
|
2691
|
+
const storedMarker = parseSha256Hex(rowValue2(row, "purge_marker_sha256", 5));
|
|
2692
|
+
const generations = integer2(rowValue2(row, "generation_count", 6));
|
|
2693
|
+
const memberships = integer2(rowValue2(row, "membership_count", 7));
|
|
2694
|
+
const orphanVectors = integer2(rowValue2(row, "orphan_vector_count", 8));
|
|
2695
|
+
const isolationScopes = integer2(rowValue2(row, "isolation_scope_count", 9));
|
|
2696
|
+
const countsRecordedValue = integer2(rowValue2(row, "counts_recorded", 10));
|
|
2697
|
+
if (isolation === null || profileSha256 !== OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256 || publishedGeneration !== null && publishedGeneration < 0 || publishedGeneration === null !== (publishedGenerationSha256 === null) || purgedAt === null || storedMarker === null || storedMarker !== purgeMarkerSha256(authorityId, isolation, purgedAt) || generations === null || generations < 0 || memberships === null || memberships < 0 || orphanVectors === null || orphanVectors < 0 || isolationScopes === null || isolationScopes < 1 || countsRecordedValue !== 0 && countsRecordedValue !== 1) {
|
|
2698
|
+
throw new OhLibSqlSemanticV2Error("integrity", "The semantic purge marker is invalid.");
|
|
2699
|
+
}
|
|
2700
|
+
return purgeResult(Object.freeze({
|
|
2701
|
+
authorityId,
|
|
2702
|
+
countsRecorded: countsRecordedValue === 1,
|
|
2703
|
+
generations,
|
|
2704
|
+
isolationScopes,
|
|
2705
|
+
isolationSha256: isolation,
|
|
2706
|
+
memberships,
|
|
2707
|
+
orphanVectors,
|
|
2708
|
+
profileSha256,
|
|
2709
|
+
publishedGeneration,
|
|
2710
|
+
publishedGenerationSha256,
|
|
2711
|
+
purgeMarkerSha256: storedMarker,
|
|
2712
|
+
purgedAt,
|
|
2713
|
+
residualGenerations: 0,
|
|
2714
|
+
residualMemberships: 0,
|
|
2715
|
+
residualScopedVectors: 0,
|
|
2716
|
+
v: 2
|
|
2717
|
+
}));
|
|
2718
|
+
}
|
|
2719
|
+
async function readIsolationOwner(client, isolation) {
|
|
2720
|
+
const result = await client.execute({
|
|
2721
|
+
args: [isolation],
|
|
2722
|
+
sql: "SELECT authority_id FROM oh_semantic_isolations WHERE isolation_sha256 = ?"
|
|
2723
|
+
});
|
|
2724
|
+
if (result.rows.length > 1)
|
|
2725
|
+
throw new OhLibSqlSemanticV2Error("integrity", "Duplicate semantic isolations.");
|
|
2726
|
+
const row = result.rows[0];
|
|
2727
|
+
if (row === undefined)
|
|
2728
|
+
return null;
|
|
2729
|
+
const authorityId = safeCode(rowValue2(row, "authority_id", 0), 256);
|
|
2730
|
+
if (authorityId === null)
|
|
2731
|
+
throw new OhLibSqlSemanticV2Error("integrity", "A semantic isolation is invalid.");
|
|
2732
|
+
return authorityId;
|
|
2733
|
+
}
|
|
2734
|
+
async function reserveIsolation(client, authorityId, isolation, createdAt) {
|
|
2735
|
+
if (await readPurge2(client, authorityId) !== null) {
|
|
2736
|
+
throw new OhLibSqlSemanticV2Error("purged", "The semantic authority was purged.");
|
|
2737
|
+
}
|
|
2738
|
+
await client.execute({
|
|
2739
|
+
args: [isolation, authorityId, createdAt, authorityId],
|
|
2740
|
+
sql: `INSERT INTO oh_semantic_isolations(isolation_sha256, authority_id, created_at)
|
|
2741
|
+
SELECT ?, ?, ?
|
|
2742
|
+
WHERE NOT EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = ?)
|
|
2743
|
+
ON CONFLICT DO NOTHING`
|
|
2744
|
+
});
|
|
2745
|
+
const owner = await readIsolationOwner(client, isolation);
|
|
2746
|
+
if (await readPurge2(client, authorityId) !== null) {
|
|
2747
|
+
throw new OhLibSqlSemanticV2Error("purged", "The semantic authority was purged.");
|
|
2748
|
+
}
|
|
2749
|
+
if (owner !== authorityId) {
|
|
2750
|
+
throw new OhLibSqlSemanticV2Error("conflict", "The semantic isolation belongs to another authority.");
|
|
2751
|
+
}
|
|
2752
|
+
}
|
|
2753
|
+
async function reservePurgeIsolation(client, authorityId, isolation, createdAt) {
|
|
2754
|
+
const head = await readHead2(client, authorityId);
|
|
2755
|
+
if (head !== null && head.isolationSha256 !== isolation) {
|
|
2756
|
+
throw new OhLibSqlSemanticV2Error("conflict", "The semantic purge isolation conflicts.");
|
|
2757
|
+
}
|
|
2758
|
+
const owner = await readIsolationOwner(client, isolation);
|
|
2759
|
+
if (owner !== null && owner !== authorityId) {
|
|
2760
|
+
throw new OhLibSqlSemanticV2Error("conflict", "The semantic isolation belongs to another authority.");
|
|
2761
|
+
}
|
|
2762
|
+
if (owner === null) {
|
|
2763
|
+
const existing = await client.execute({
|
|
2764
|
+
args: [authorityId],
|
|
2765
|
+
sql: "SELECT isolation_sha256 FROM oh_semantic_isolations WHERE authority_id = ? LIMIT 1"
|
|
2766
|
+
});
|
|
2767
|
+
if (existing.rows.length !== 0) {
|
|
2768
|
+
throw new OhLibSqlSemanticV2Error("conflict", "The semantic purge isolation conflicts.");
|
|
2769
|
+
}
|
|
2770
|
+
await reserveIsolation(client, authorityId, isolation, createdAt);
|
|
2771
|
+
}
|
|
2772
|
+
}
|
|
2773
|
+
async function verifyPurgeResidual(client, authorityId) {
|
|
2774
|
+
const result = await client.execute({
|
|
2775
|
+
args: [authorityId, authorityId, authorityId, authorityId],
|
|
2776
|
+
sql: `SELECT
|
|
2777
|
+
(SELECT count(*) FROM oh_semantic_heads WHERE authority_id = ?) AS heads,
|
|
2778
|
+
(SELECT count(*) FROM oh_semantic_generations WHERE authority_id = ?) AS generations,
|
|
2779
|
+
(SELECT count(*) FROM oh_semantic_memberships WHERE authority_id = ?) AS memberships,
|
|
2780
|
+
(SELECT count(*) FROM oh_semantic_vectors AS vector
|
|
2781
|
+
JOIN oh_semantic_isolations AS isolation
|
|
2782
|
+
ON isolation.isolation_sha256 = vector.isolation_sha256
|
|
2783
|
+
WHERE isolation.authority_id = ?) AS vectors`
|
|
2784
|
+
});
|
|
2785
|
+
const row = result.rows[0];
|
|
2786
|
+
if (result.rows.length !== 1 || row === undefined || integer2(rowValue2(row, "heads", 0)) !== 0 || integer2(rowValue2(row, "generations", 1)) !== 0 || integer2(rowValue2(row, "memberships", 2)) !== 0 || integer2(rowValue2(row, "vectors", 3)) !== 0) {
|
|
2787
|
+
throw new OhLibSqlSemanticV2Error("integrity", "The semantic authority purge is incomplete.");
|
|
2788
|
+
}
|
|
2789
|
+
}
|
|
2790
|
+
async function readMemberships2(client, generation) {
|
|
2791
|
+
const memberships = [];
|
|
2792
|
+
for (let offset = 0;offset < generation.chunkCount; offset += OH_LIBSQL_SEMANTIC_LIMITS_V2.searchPage) {
|
|
2793
|
+
const result = await client.execute({
|
|
2794
|
+
args: [
|
|
2795
|
+
generation.authorityId,
|
|
2796
|
+
generation.generation,
|
|
2797
|
+
OH_LIBSQL_SEMANTIC_LIMITS_V2.searchPage,
|
|
2798
|
+
offset
|
|
2799
|
+
],
|
|
2800
|
+
sql: `SELECT generation_sha256, isolation_sha256, record_key,
|
|
2801
|
+
record_sha256, ordinal, input_sha256
|
|
2802
|
+
FROM oh_semantic_memberships
|
|
2803
|
+
WHERE authority_id = ? AND generation = ?
|
|
2804
|
+
ORDER BY record_key, ordinal LIMIT ? OFFSET ?`
|
|
2805
|
+
});
|
|
2806
|
+
for (const row of result.rows) {
|
|
2807
|
+
const generationSha256 = parseSha256Hex(rowValue2(row, "generation_sha256", 0));
|
|
2808
|
+
const isolation = parseSha256Hex(rowValue2(row, "isolation_sha256", 1));
|
|
2809
|
+
const recordKey2 = safeCode(rowValue2(row, "record_key", 2), 512);
|
|
2810
|
+
const recordSha256 = parseSha256Hex(rowValue2(row, "record_sha256", 3));
|
|
2811
|
+
const ordinal = integer2(rowValue2(row, "ordinal", 4));
|
|
2812
|
+
const inputSha256 = parseSha256Hex(rowValue2(row, "input_sha256", 5));
|
|
2813
|
+
if (generationSha256 !== generation.generationSha256 || isolation !== generation.isolationSha256 || recordKey2 === null || recordSha256 === null || ordinal === null || ordinal < 0 || ordinal >= OH_LIBSQL_SEMANTIC_LIMITS_V2.chunksPerDocument || inputSha256 === null) {
|
|
2814
|
+
throw new OhLibSqlSemanticV2Error("integrity", "A semantic generation membership is invalid.");
|
|
2815
|
+
}
|
|
2816
|
+
memberships.push(Object.freeze({ inputSha256, ordinal, recordKey: recordKey2, recordSha256 }));
|
|
2817
|
+
}
|
|
2818
|
+
}
|
|
2819
|
+
if (memberships.length !== generation.chunkCount || canonicalSha256({
|
|
2820
|
+
kind: MEMBERSHIP_KIND,
|
|
2821
|
+
memberships: memberships.map((membership) => ({
|
|
2822
|
+
inputSha256: membership.inputSha256,
|
|
2823
|
+
isolationSha256: generation.isolationSha256,
|
|
2824
|
+
ordinal: membership.ordinal,
|
|
2825
|
+
recordKey: membership.recordKey,
|
|
2826
|
+
recordSha256: membership.recordSha256
|
|
2827
|
+
})),
|
|
2828
|
+
v: 2
|
|
2829
|
+
}) !== generation.membershipSha256) {
|
|
2830
|
+
throw new OhLibSqlSemanticV2Error("integrity", "A semantic generation membership digest is invalid.");
|
|
2831
|
+
}
|
|
2832
|
+
return Object.freeze(memberships);
|
|
2833
|
+
}
|
|
2834
|
+
async function readVectors2(client, isolationSha2562, inputSha256s) {
|
|
2835
|
+
const vectors = new Map;
|
|
2836
|
+
for (let offset = 0;offset < inputSha256s.length; offset += 64) {
|
|
2837
|
+
const page = inputSha256s.slice(offset, offset + 64);
|
|
2838
|
+
if (page.length === 0)
|
|
2839
|
+
continue;
|
|
2840
|
+
const placeholders = page.map(() => "?").join(", ");
|
|
2841
|
+
const result = await client.execute({
|
|
2842
|
+
args: [
|
|
2843
|
+
isolationSha2562,
|
|
2844
|
+
OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256,
|
|
2845
|
+
OH_SEMANTIC_RENDERER_V1.rendererSha256,
|
|
2846
|
+
...page
|
|
2847
|
+
],
|
|
2848
|
+
sql: `SELECT input_sha256, vector_sha256, vector FROM oh_semantic_vectors
|
|
2849
|
+
WHERE isolation_sha256 = ? AND profile_sha256 = ? AND renderer_sha256 = ?
|
|
2850
|
+
AND input_sha256 IN (${placeholders}) ORDER BY input_sha256`
|
|
2851
|
+
});
|
|
2852
|
+
for (const row of result.rows) {
|
|
2853
|
+
const inputSha256 = parseSha256Hex(rowValue2(row, "input_sha256", 0));
|
|
2854
|
+
const vectorSha256 = parseSha256Hex(rowValue2(row, "vector_sha256", 1));
|
|
2855
|
+
if (inputSha256 === null || vectorSha256 === null || !page.includes(inputSha256) || vectors.has(inputSha256)) {
|
|
2856
|
+
throw new OhLibSqlSemanticV2Error("integrity", "A cached semantic vector identity is invalid.");
|
|
2857
|
+
}
|
|
2858
|
+
const bytes = storedBytes2(rowValue2(row, "vector", 2));
|
|
2859
|
+
if (bytes === null) {
|
|
2860
|
+
throw new OhLibSqlSemanticV2Error("integrity", "A cached semantic vector is corrupt.");
|
|
2861
|
+
}
|
|
2862
|
+
decodeVector2(bytes, vectorSha256);
|
|
2863
|
+
vectors.set(inputSha256, Object.freeze({
|
|
2864
|
+
bytes,
|
|
2865
|
+
inputSha256,
|
|
2866
|
+
vectorSha256
|
|
2867
|
+
}));
|
|
2868
|
+
}
|
|
2869
|
+
}
|
|
2870
|
+
return vectors;
|
|
2871
|
+
}
|
|
2872
|
+
function validateEmbeddingClient2(client) {
|
|
2873
|
+
if (client.profile.profileSha256 !== OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256) {
|
|
2874
|
+
throw new OhLibSqlSemanticV2Error("invalid-input", "The embedding client profile is incompatible.");
|
|
2875
|
+
}
|
|
2876
|
+
}
|
|
2877
|
+
|
|
2878
|
+
class OhLibSqlSemanticCacheV2 {
|
|
2879
|
+
#client;
|
|
2880
|
+
#closeClient;
|
|
2881
|
+
#closed = false;
|
|
2882
|
+
constructor(client, closeClient) {
|
|
2883
|
+
this.#client = client;
|
|
2884
|
+
this.#closeClient = closeClient;
|
|
2885
|
+
}
|
|
2886
|
+
static async open(client, closeClient) {
|
|
2887
|
+
await verifySchema2(client);
|
|
2888
|
+
return new OhLibSqlSemanticCacheV2(client, closeClient);
|
|
2889
|
+
}
|
|
2890
|
+
#open() {
|
|
2891
|
+
if (this.#closed)
|
|
2892
|
+
throw new OhLibSqlSemanticV2Error("schema-unavailable", "The semantic cache is closed.");
|
|
2893
|
+
}
|
|
2894
|
+
async close() {
|
|
2895
|
+
if (this.#closed)
|
|
2896
|
+
return;
|
|
2897
|
+
this.#closed = true;
|
|
2898
|
+
if (this.#closeClient)
|
|
2899
|
+
this.#client.close?.();
|
|
2900
|
+
}
|
|
2901
|
+
async publishedHead(input) {
|
|
2902
|
+
this.#open();
|
|
2903
|
+
const authorityId = parseAuthorityId2(input.authorityId);
|
|
2904
|
+
const isolation = isolationSha256(authorityId, input.isolationSha256);
|
|
2905
|
+
if (await readPurge2(this.#client, authorityId) !== null)
|
|
2906
|
+
return null;
|
|
2907
|
+
const head = await readHead2(this.#client, authorityId);
|
|
2908
|
+
if (head === null || head.isolationSha256 !== isolation)
|
|
2909
|
+
return null;
|
|
2910
|
+
const generation = await readGeneration2(this.#client, authorityId, head.generation);
|
|
2911
|
+
if (generation === null || !headMatchesGeneration2(head, generation)) {
|
|
2912
|
+
if (await readPurge2(this.#client, authorityId) !== null)
|
|
2913
|
+
return null;
|
|
2914
|
+
throw new OhLibSqlSemanticV2Error("integrity", "The semantic published head does not match its immutable generation.");
|
|
2915
|
+
}
|
|
2916
|
+
if (await readPurge2(this.#client, authorityId) !== null)
|
|
2917
|
+
return null;
|
|
2918
|
+
const finalHead = await readHead2(this.#client, authorityId);
|
|
2919
|
+
if (finalHead === null) {
|
|
2920
|
+
if (await readPurge2(this.#client, authorityId) !== null)
|
|
2921
|
+
return null;
|
|
2922
|
+
throw new OhLibSqlSemanticV2Error("integrity", "The semantic published head disappeared during its read.");
|
|
2923
|
+
}
|
|
2924
|
+
if (canonicalJson(finalHead) !== canonicalJson(head)) {
|
|
2925
|
+
throw new OhLibSqlSemanticV2Error("conflict", "The semantic published head changed during its read.");
|
|
2926
|
+
}
|
|
2927
|
+
return Object.freeze({ ...head, v: 2 });
|
|
2928
|
+
}
|
|
2929
|
+
async stage(input) {
|
|
2930
|
+
this.#open();
|
|
2931
|
+
validateEmbeddingClient2(input.embeddingClient);
|
|
2932
|
+
const prepared = prepareGeneration2(input);
|
|
2933
|
+
await reserveIsolation(this.#client, prepared.authorityId, prepared.isolationSha256, prepared.createdAt);
|
|
2934
|
+
if (await readPurge2(this.#client, prepared.authorityId) !== null) {
|
|
2935
|
+
throw new OhLibSqlSemanticV2Error("purged", "The semantic authority was purged.");
|
|
2936
|
+
}
|
|
2937
|
+
const existingGeneration = await readGeneration2(this.#client, prepared.authorityId, prepared.generation);
|
|
2938
|
+
if (existingGeneration !== null && !generationMatches2(existingGeneration, prepared)) {
|
|
2939
|
+
throw new OhLibSqlSemanticV2Error("conflict", "The semantic generation identity conflicts.");
|
|
2940
|
+
}
|
|
2941
|
+
const uniqueInputs = new Map;
|
|
2942
|
+
for (const membership of prepared.memberships)
|
|
2943
|
+
uniqueInputs.set(membership.inputSha256, membership.input);
|
|
2944
|
+
const orderedInputs = [...uniqueInputs.entries()].sort(([left], [right]) => left < right ? -1 : 1);
|
|
2945
|
+
const existingVectors = await readVectors2(this.#client, prepared.isolationSha256, orderedInputs.map(([digest]) => digest));
|
|
2946
|
+
const missing = orderedInputs.filter(([digest]) => !existingVectors.has(digest));
|
|
2947
|
+
const candidateVectors = new Map(existingVectors);
|
|
2948
|
+
for (let offset = 0;offset < missing.length; offset += OH_LIBSQL_SEMANTIC_LIMITS_V2.embeddingBatch) {
|
|
2949
|
+
const page = missing.slice(offset, offset + OH_LIBSQL_SEMANTIC_LIMITS_V2.embeddingBatch);
|
|
2950
|
+
const vectors = await input.embeddingClient.embed(page.map(([, rendered]) => rendered), input.signal === undefined ? {} : { signal: input.signal });
|
|
2951
|
+
if (vectors.length !== page.length) {
|
|
2952
|
+
throw new OhLibSqlSemanticV2Error("integrity", "The embedding client returned a mismatched vector batch.");
|
|
2953
|
+
}
|
|
2954
|
+
for (const [index, [inputSha256]] of page.entries()) {
|
|
2955
|
+
const vector = vectors[index];
|
|
2956
|
+
if (vector === undefined)
|
|
2957
|
+
throw new OhLibSqlSemanticV2Error("integrity", "A semantic vector is missing.");
|
|
2958
|
+
const bytes = vectorBytes2(vector);
|
|
2959
|
+
const vectorSha256 = sha256Hex(bytes);
|
|
2960
|
+
decodeVector2(bytes, vectorSha256);
|
|
2961
|
+
candidateVectors.set(inputSha256, Object.freeze({ bytes, inputSha256, vectorSha256 }));
|
|
2962
|
+
}
|
|
2963
|
+
}
|
|
2964
|
+
const statements = [{
|
|
2965
|
+
args: [
|
|
2966
|
+
prepared.authorityId,
|
|
2967
|
+
prepared.generation,
|
|
2968
|
+
prepared.authoritySha256,
|
|
2969
|
+
prepared.isolationSha256,
|
|
2970
|
+
OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256,
|
|
2971
|
+
OH_SEMANTIC_RENDERER_V1.rendererSha256,
|
|
2972
|
+
prepared.membershipSha256,
|
|
2973
|
+
prepared.generationSha256,
|
|
2974
|
+
prepared.documentCount,
|
|
2975
|
+
prepared.chunkCount,
|
|
2976
|
+
prepared.createdAt,
|
|
2977
|
+
prepared.authorityId
|
|
2978
|
+
],
|
|
2979
|
+
sql: `INSERT INTO oh_semantic_generations(authority_id, generation,
|
|
2980
|
+
authority_sha256, isolation_sha256, profile_sha256, renderer_sha256, membership_sha256,
|
|
2981
|
+
generation_sha256, document_count, chunk_count, created_at)
|
|
2982
|
+
SELECT ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
|
|
2983
|
+
WHERE NOT EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = ?)
|
|
2984
|
+
ON CONFLICT DO NOTHING`
|
|
2985
|
+
}];
|
|
2986
|
+
for (const [inputSha256] of orderedInputs) {
|
|
2987
|
+
const candidate = candidateVectors.get(inputSha256);
|
|
2988
|
+
if (candidate === undefined) {
|
|
2989
|
+
throw new OhLibSqlSemanticV2Error("integrity", "A semantic vector candidate is missing.");
|
|
2990
|
+
}
|
|
2991
|
+
statements.push({
|
|
2992
|
+
args: [
|
|
2993
|
+
prepared.isolationSha256,
|
|
2994
|
+
OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256,
|
|
2995
|
+
OH_SEMANTIC_RENDERER_V1.rendererSha256,
|
|
2996
|
+
inputSha256,
|
|
2997
|
+
candidate.vectorSha256,
|
|
2998
|
+
candidate.bytes,
|
|
2999
|
+
prepared.createdAt,
|
|
3000
|
+
prepared.authorityId,
|
|
3001
|
+
prepared.generation,
|
|
3002
|
+
prepared.generationSha256,
|
|
3003
|
+
prepared.isolationSha256,
|
|
3004
|
+
prepared.authorityId
|
|
3005
|
+
],
|
|
3006
|
+
sql: `INSERT INTO oh_semantic_vectors(isolation_sha256, profile_sha256,
|
|
3007
|
+
renderer_sha256, input_sha256, vector_sha256, vector, created_at)
|
|
3008
|
+
SELECT ?, ?, ?, ?, ?, ?, ?
|
|
3009
|
+
WHERE EXISTS (SELECT 1 FROM oh_semantic_generations
|
|
3010
|
+
WHERE authority_id = ? AND generation = ? AND generation_sha256 = ?
|
|
3011
|
+
AND isolation_sha256 = ?)
|
|
3012
|
+
AND NOT EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = ?)
|
|
3013
|
+
ON CONFLICT DO NOTHING`
|
|
3014
|
+
});
|
|
3015
|
+
}
|
|
3016
|
+
for (const membership of prepared.memberships) {
|
|
3017
|
+
statements.push({
|
|
3018
|
+
args: [
|
|
3019
|
+
prepared.authorityId,
|
|
3020
|
+
prepared.generation,
|
|
3021
|
+
prepared.generationSha256,
|
|
3022
|
+
prepared.isolationSha256,
|
|
3023
|
+
membership.recordKey,
|
|
3024
|
+
membership.recordSha256,
|
|
3025
|
+
membership.ordinal,
|
|
3026
|
+
membership.inputSha256,
|
|
3027
|
+
prepared.authorityId,
|
|
3028
|
+
prepared.generation,
|
|
3029
|
+
prepared.generationSha256,
|
|
3030
|
+
prepared.isolationSha256,
|
|
3031
|
+
prepared.authorityId
|
|
3032
|
+
],
|
|
3033
|
+
sql: `INSERT INTO oh_semantic_memberships(authority_id, generation,
|
|
3034
|
+
generation_sha256, isolation_sha256, record_key, record_sha256, ordinal, input_sha256)
|
|
3035
|
+
SELECT ?, ?, ?, ?, ?, ?, ?, ?
|
|
3036
|
+
WHERE EXISTS (SELECT 1 FROM oh_semantic_generations
|
|
3037
|
+
WHERE authority_id = ? AND generation = ? AND generation_sha256 = ?
|
|
3038
|
+
AND isolation_sha256 = ?)
|
|
3039
|
+
AND NOT EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = ?)
|
|
3040
|
+
ON CONFLICT DO NOTHING`
|
|
3041
|
+
});
|
|
3042
|
+
}
|
|
3043
|
+
await this.#client.batch(statements, "write");
|
|
3044
|
+
const completeVectors = await readVectors2(this.#client, prepared.isolationSha256, orderedInputs.map(([digest]) => digest));
|
|
3045
|
+
if (completeVectors.size !== orderedInputs.length) {
|
|
3046
|
+
if (await readPurge2(this.#client, prepared.authorityId) !== null) {
|
|
3047
|
+
throw new OhLibSqlSemanticV2Error("purged", "The semantic authority was purged.");
|
|
3048
|
+
}
|
|
3049
|
+
throw new OhLibSqlSemanticV2Error("integrity", "The semantic vector cache did not converge.");
|
|
3050
|
+
}
|
|
3051
|
+
const stored = await readGeneration2(this.#client, prepared.authorityId, prepared.generation);
|
|
3052
|
+
if (await readPurge2(this.#client, prepared.authorityId) !== null) {
|
|
3053
|
+
throw new OhLibSqlSemanticV2Error("purged", "The semantic authority was purged.");
|
|
3054
|
+
}
|
|
3055
|
+
if (stored === null || !generationMatches2(stored, prepared)) {
|
|
3056
|
+
throw new OhLibSqlSemanticV2Error("conflict", "The semantic generation identity conflicts.");
|
|
3057
|
+
}
|
|
3058
|
+
const memberships = await readMemberships2(this.#client, stored);
|
|
3059
|
+
if (canonicalJson(memberships) !== canonicalJson(prepared.memberships.map((membership) => ({
|
|
3060
|
+
inputSha256: membership.inputSha256,
|
|
3061
|
+
ordinal: membership.ordinal,
|
|
3062
|
+
recordKey: membership.recordKey,
|
|
3063
|
+
recordSha256: membership.recordSha256
|
|
3064
|
+
})))) {
|
|
3065
|
+
throw new OhLibSqlSemanticV2Error("conflict", "The semantic generation membership conflicts.");
|
|
3066
|
+
}
|
|
3067
|
+
return Object.freeze({
|
|
3068
|
+
authorityId: prepared.authorityId,
|
|
3069
|
+
chunks: prepared.chunkCount,
|
|
3070
|
+
documents: prepared.documentCount,
|
|
3071
|
+
embedded: missing.length,
|
|
3072
|
+
generation: prepared.generation,
|
|
3073
|
+
generationSha256: prepared.generationSha256,
|
|
3074
|
+
isolationSha256: prepared.isolationSha256,
|
|
3075
|
+
membershipSha256: prepared.membershipSha256,
|
|
3076
|
+
reused: orderedInputs.length - missing.length,
|
|
3077
|
+
status: "staged",
|
|
3078
|
+
v: 2
|
|
3079
|
+
});
|
|
3080
|
+
}
|
|
3081
|
+
async publish(input) {
|
|
3082
|
+
this.#open();
|
|
3083
|
+
const authorityId = parseAuthorityId2(input.authorityId);
|
|
3084
|
+
const isolation = isolationSha256(authorityId, input.isolationSha256);
|
|
3085
|
+
const generationNumber = parseGeneration2(input.generation);
|
|
3086
|
+
const expected = input.expectedPublishedGeneration === null ? null : parseGeneration2(input.expectedPublishedGeneration);
|
|
3087
|
+
const publishedAt = parseInstant2(input.publishedAt ?? canonicalNow());
|
|
3088
|
+
if (await readPurge2(this.#client, authorityId) !== null) {
|
|
3089
|
+
throw new OhLibSqlSemanticV2Error("purged", "The semantic authority was purged.");
|
|
3090
|
+
}
|
|
3091
|
+
const generation = await readGeneration2(this.#client, authorityId, generationNumber);
|
|
3092
|
+
if (generation === null || generation.isolationSha256 !== isolation) {
|
|
3093
|
+
throw new OhLibSqlSemanticV2Error("conflict", "The semantic generation is not staged.");
|
|
3094
|
+
}
|
|
3095
|
+
await readMemberships2(this.#client, generation);
|
|
3096
|
+
const before = await readHead2(this.#client, authorityId);
|
|
3097
|
+
if (before !== null && headMatchesGeneration2(before, generation)) {
|
|
3098
|
+
return Object.freeze({
|
|
3099
|
+
authorityId,
|
|
3100
|
+
generation: generationNumber,
|
|
3101
|
+
generationSha256: generation.generationSha256,
|
|
3102
|
+
isolationSha256: isolation,
|
|
3103
|
+
published: false,
|
|
3104
|
+
v: 2
|
|
3105
|
+
});
|
|
3106
|
+
}
|
|
3107
|
+
if (before === null !== (expected === null) || before !== null && before.generation !== expected || before !== null && generationNumber < before.generation) {
|
|
3108
|
+
throw new OhLibSqlSemanticV2Error("conflict", "The semantic published-head precondition failed.");
|
|
3109
|
+
}
|
|
3110
|
+
let result;
|
|
3111
|
+
const values = [
|
|
3112
|
+
generation.authorityId,
|
|
3113
|
+
generation.generation,
|
|
3114
|
+
generation.authoritySha256,
|
|
3115
|
+
generation.isolationSha256,
|
|
3116
|
+
generation.profileSha256,
|
|
3117
|
+
generation.rendererSha256,
|
|
3118
|
+
generation.membershipSha256,
|
|
3119
|
+
generation.generationSha256,
|
|
3120
|
+
publishedAt
|
|
3121
|
+
];
|
|
3122
|
+
if (expected === null) {
|
|
3123
|
+
result = await this.#client.execute({
|
|
3124
|
+
args: [
|
|
3125
|
+
...values,
|
|
3126
|
+
generation.authorityId,
|
|
3127
|
+
generation.generation,
|
|
3128
|
+
generation.generationSha256,
|
|
3129
|
+
generation.isolationSha256,
|
|
3130
|
+
authorityId,
|
|
3131
|
+
authorityId
|
|
3132
|
+
],
|
|
3133
|
+
sql: `INSERT INTO oh_semantic_heads(authority_id, generation,
|
|
3134
|
+
authority_sha256, isolation_sha256, profile_sha256, renderer_sha256, membership_sha256,
|
|
3135
|
+
generation_sha256, published_at)
|
|
3136
|
+
SELECT ?, ?, ?, ?, ?, ?, ?, ?, ?
|
|
3137
|
+
WHERE EXISTS (SELECT 1 FROM oh_semantic_generations
|
|
3138
|
+
WHERE authority_id = ? AND generation = ? AND generation_sha256 = ?
|
|
3139
|
+
AND isolation_sha256 = ?)
|
|
3140
|
+
AND NOT EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = ?)
|
|
3141
|
+
AND NOT EXISTS (SELECT 1 FROM oh_semantic_heads WHERE authority_id = ?)
|
|
3142
|
+
ON CONFLICT DO NOTHING`
|
|
3143
|
+
});
|
|
3144
|
+
} else {
|
|
3145
|
+
result = await this.#client.execute({
|
|
3146
|
+
args: [
|
|
3147
|
+
generation.generation,
|
|
3148
|
+
generation.authoritySha256,
|
|
3149
|
+
generation.isolationSha256,
|
|
3150
|
+
generation.profileSha256,
|
|
3151
|
+
generation.rendererSha256,
|
|
3152
|
+
generation.membershipSha256,
|
|
3153
|
+
generation.generationSha256,
|
|
3154
|
+
publishedAt,
|
|
3155
|
+
authorityId,
|
|
3156
|
+
expected,
|
|
3157
|
+
generation.authorityId,
|
|
3158
|
+
generation.generation,
|
|
3159
|
+
generation.generationSha256,
|
|
3160
|
+
generation.isolationSha256,
|
|
3161
|
+
authorityId
|
|
3162
|
+
],
|
|
3163
|
+
sql: `UPDATE oh_semantic_heads SET generation = ?, authority_sha256 = ?,
|
|
3164
|
+
isolation_sha256 = ?, profile_sha256 = ?, renderer_sha256 = ?, membership_sha256 = ?,
|
|
3165
|
+
generation_sha256 = ?, published_at = ?
|
|
3166
|
+
WHERE authority_id = ? AND generation = ?
|
|
3167
|
+
AND EXISTS (SELECT 1 FROM oh_semantic_generations
|
|
3168
|
+
WHERE authority_id = ? AND generation = ? AND generation_sha256 = ?
|
|
3169
|
+
AND isolation_sha256 = ?)
|
|
3170
|
+
AND NOT EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = ?)`
|
|
3171
|
+
});
|
|
3172
|
+
}
|
|
3173
|
+
if (await readPurge2(this.#client, authorityId) !== null) {
|
|
3174
|
+
throw new OhLibSqlSemanticV2Error("purged", "The semantic authority was purged.");
|
|
3175
|
+
}
|
|
3176
|
+
const after = await readHead2(this.#client, authorityId);
|
|
3177
|
+
if (after === null || !headMatchesGeneration2(after, generation)) {
|
|
3178
|
+
throw new OhLibSqlSemanticV2Error("conflict", "The semantic published head did not converge.");
|
|
3179
|
+
}
|
|
3180
|
+
return Object.freeze({
|
|
3181
|
+
authorityId,
|
|
3182
|
+
generation: generationNumber,
|
|
3183
|
+
generationSha256: generation.generationSha256,
|
|
3184
|
+
isolationSha256: isolation,
|
|
3185
|
+
published: rowsAffected2(result) > 0,
|
|
3186
|
+
v: 2
|
|
3187
|
+
});
|
|
3188
|
+
}
|
|
3189
|
+
async search(input) {
|
|
3190
|
+
this.#open();
|
|
3191
|
+
validateEmbeddingClient2(input.embeddingClient);
|
|
3192
|
+
const authorityId = parseAuthorityId2(input.authority.authorityId);
|
|
3193
|
+
const authoritySha256 = parseDigest2(input.authority.authoritySha256, "authority");
|
|
3194
|
+
const isolation = isolationSha256(authorityId, input.authority.isolationSha256);
|
|
3195
|
+
const authorityGeneration = parseGeneration2(input.authority.generation);
|
|
3196
|
+
const limit = input.limit ?? 10;
|
|
3197
|
+
if (input.authority.v !== 2 || !Number.isSafeInteger(limit) || limit < 1 || limit > OH_LIBSQL_SEMANTIC_LIMITS_V2.searchLimit || !Array.isArray(input.authority.records) || input.authority.records.length > OH_LIBSQL_SEMANTIC_LIMITS_V2.documentsPerGeneration) {
|
|
3198
|
+
throw new OhLibSqlSemanticV2Error("invalid-input", "Invalid semantic search authority or limit.");
|
|
3199
|
+
}
|
|
3200
|
+
const records = new Map;
|
|
3201
|
+
for (const record of input.authority.records) {
|
|
3202
|
+
const key = parseRecordKey2(record.key);
|
|
3203
|
+
const recordSha256 = parseDigest2(record.recordSha256, "record");
|
|
3204
|
+
if (records.has(key))
|
|
3205
|
+
throw new OhLibSqlSemanticV2Error("invalid-input", "Duplicate authority record key.");
|
|
3206
|
+
records.set(key, recordSha256);
|
|
3207
|
+
}
|
|
3208
|
+
if (await readPurge2(this.#client, authorityId) !== null)
|
|
3209
|
+
return Object.freeze([]);
|
|
3210
|
+
const head = await readHead2(this.#client, authorityId);
|
|
3211
|
+
if (head === null || head.authoritySha256 !== authoritySha256 || head.generation !== authorityGeneration || head.isolationSha256 !== isolation || head.profileSha256 !== OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256 || head.rendererSha256 !== OH_SEMANTIC_RENDERER_V1.rendererSha256) {
|
|
3212
|
+
return Object.freeze([]);
|
|
3213
|
+
}
|
|
3214
|
+
const generation = await readGeneration2(this.#client, authorityId, authorityGeneration);
|
|
3215
|
+
if (generation === null || !headMatchesGeneration2(head, generation))
|
|
3216
|
+
return Object.freeze([]);
|
|
3217
|
+
const renderedQuery = renderOhCloudflareEmbeddingQueryV1(input.query);
|
|
3218
|
+
const queryVectors = await input.embeddingClient.embed([renderedQuery], input.signal === undefined ? {} : { signal: input.signal });
|
|
3219
|
+
const queryVector = queryVectors[0];
|
|
3220
|
+
if (queryVectors.length !== 1 || queryVector === undefined) {
|
|
3221
|
+
throw new OhLibSqlSemanticV2Error("integrity", "The query embedding response is invalid.");
|
|
3222
|
+
}
|
|
3223
|
+
const normalizedQuery = normalizeOhEmbeddingV1(queryVector);
|
|
3224
|
+
const best = new Map;
|
|
3225
|
+
let scanned = 0;
|
|
3226
|
+
for (let offset = 0;offset < generation.chunkCount; offset += OH_LIBSQL_SEMANTIC_LIMITS_V2.searchPage) {
|
|
3227
|
+
const result = await this.#client.execute({
|
|
3228
|
+
args: [
|
|
3229
|
+
OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256,
|
|
3230
|
+
OH_SEMANTIC_RENDERER_V1.rendererSha256,
|
|
3231
|
+
authorityId,
|
|
3232
|
+
authorityGeneration,
|
|
3233
|
+
isolation,
|
|
3234
|
+
OH_LIBSQL_SEMANTIC_LIMITS_V2.searchPage,
|
|
3235
|
+
offset
|
|
3236
|
+
],
|
|
3237
|
+
sql: `SELECT membership.generation_sha256, membership.isolation_sha256,
|
|
3238
|
+
membership.record_key,
|
|
3239
|
+
membership.record_sha256, membership.ordinal, membership.input_sha256,
|
|
3240
|
+
vector.vector_sha256, vector.vector
|
|
3241
|
+
FROM oh_semantic_memberships AS membership
|
|
3242
|
+
JOIN oh_semantic_vectors AS vector
|
|
3243
|
+
ON vector.input_sha256 = membership.input_sha256
|
|
3244
|
+
AND vector.isolation_sha256 = membership.isolation_sha256
|
|
3245
|
+
AND vector.profile_sha256 = ? AND vector.renderer_sha256 = ?
|
|
3246
|
+
WHERE membership.authority_id = ? AND membership.generation = ?
|
|
3247
|
+
AND membership.isolation_sha256 = ?
|
|
3248
|
+
ORDER BY membership.record_key, membership.ordinal LIMIT ? OFFSET ?`
|
|
3249
|
+
});
|
|
3250
|
+
for (const row of result.rows) {
|
|
3251
|
+
scanned += 1;
|
|
3252
|
+
const generationSha256 = parseSha256Hex(rowValue2(row, "generation_sha256", 0));
|
|
3253
|
+
const storedIsolation = parseSha256Hex(rowValue2(row, "isolation_sha256", 1));
|
|
3254
|
+
const key = safeCode(rowValue2(row, "record_key", 2), 512);
|
|
3255
|
+
const recordSha256 = parseSha256Hex(rowValue2(row, "record_sha256", 3));
|
|
3256
|
+
const ordinal = integer2(rowValue2(row, "ordinal", 4));
|
|
3257
|
+
const inputSha256 = parseSha256Hex(rowValue2(row, "input_sha256", 5));
|
|
3258
|
+
const vectorSha256 = parseSha256Hex(rowValue2(row, "vector_sha256", 6));
|
|
3259
|
+
if (generationSha256 !== generation.generationSha256 || storedIsolation !== isolation || key === null || recordSha256 === null || ordinal === null || ordinal < 0 || ordinal >= OH_LIBSQL_SEMANTIC_LIMITS_V2.chunksPerDocument || inputSha256 === null || vectorSha256 === null) {
|
|
3260
|
+
throw new OhLibSqlSemanticV2Error("integrity", "A semantic search row is invalid.");
|
|
3261
|
+
}
|
|
3262
|
+
if (records.get(key) !== recordSha256)
|
|
3263
|
+
continue;
|
|
3264
|
+
const vector = decodeVector2(rowValue2(row, "vector", 7), vectorSha256);
|
|
3265
|
+
let score = 0;
|
|
3266
|
+
for (let index = 0;index < normalizedQuery.length; index += 1) {
|
|
3267
|
+
score += normalizedQuery[index] * vector[index];
|
|
3268
|
+
}
|
|
3269
|
+
score = Math.max(-1, Math.min(1, score));
|
|
3270
|
+
const previous = best.get(key);
|
|
3271
|
+
if (previous === undefined || score > previous.score || score === previous.score && ordinal < previous.chunkOrdinal) {
|
|
3272
|
+
best.set(key, Object.freeze({ chunkOrdinal: ordinal, key, recordSha256, score, v: 2 }));
|
|
3273
|
+
}
|
|
3274
|
+
}
|
|
3275
|
+
}
|
|
3276
|
+
if (scanned !== generation.chunkCount) {
|
|
3277
|
+
throw new OhLibSqlSemanticV2Error("integrity", "The semantic search scan is incomplete.");
|
|
3278
|
+
}
|
|
3279
|
+
const finalHead = await readHead2(this.#client, authorityId);
|
|
3280
|
+
if (finalHead === null || canonicalJson(finalHead) !== canonicalJson(head) || await readPurge2(this.#client, authorityId) !== null)
|
|
3281
|
+
return Object.freeze([]);
|
|
3282
|
+
return Object.freeze([...best.values()].sort((left, right) => right.score - left.score || (left.key < right.key ? -1 : left.key > right.key ? 1 : 0)).slice(0, limit));
|
|
3283
|
+
}
|
|
3284
|
+
async purgeReceipt(input) {
|
|
3285
|
+
this.#open();
|
|
3286
|
+
const authorityId = parseAuthorityId2(input.authorityId);
|
|
3287
|
+
const isolation = isolationSha256(authorityId, input.isolationSha256);
|
|
3288
|
+
const receipt = await readPurge2(this.#client, authorityId);
|
|
3289
|
+
if (receipt === null)
|
|
3290
|
+
return null;
|
|
3291
|
+
if (receipt.isolationSha256 !== isolation) {
|
|
3292
|
+
throw new OhLibSqlSemanticV2Error("conflict", "The semantic purge isolation conflicts.");
|
|
3293
|
+
}
|
|
3294
|
+
await verifyPurgeResidual(this.#client, authorityId);
|
|
3295
|
+
return receipt;
|
|
3296
|
+
}
|
|
3297
|
+
async purgeAuthority(input) {
|
|
3298
|
+
this.#open();
|
|
3299
|
+
const authorityId = parseAuthorityId2(input.authorityId);
|
|
3300
|
+
const isolation = isolationSha256(authorityId, input.isolationSha256);
|
|
3301
|
+
const previous = await readPurge2(this.#client, authorityId);
|
|
3302
|
+
if (previous !== null) {
|
|
3303
|
+
if (previous.isolationSha256 !== isolation) {
|
|
3304
|
+
throw new OhLibSqlSemanticV2Error("conflict", "The semantic purge isolation conflicts.");
|
|
3305
|
+
}
|
|
3306
|
+
await verifyPurgeResidual(this.#client, authorityId);
|
|
3307
|
+
return previous;
|
|
3308
|
+
}
|
|
3309
|
+
const requestedAt = parseInstant2(input.purgedAt ?? canonicalNow());
|
|
3310
|
+
await reservePurgeIsolation(this.#client, authorityId, isolation, requestedAt);
|
|
3311
|
+
const markerSha256 = purgeMarkerSha256(authorityId, isolation, requestedAt);
|
|
3312
|
+
await this.#client.batch([
|
|
3313
|
+
{
|
|
3314
|
+
args: [
|
|
3315
|
+
authorityId,
|
|
3316
|
+
isolation,
|
|
3317
|
+
OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256,
|
|
3318
|
+
authorityId,
|
|
3319
|
+
authorityId,
|
|
3320
|
+
requestedAt,
|
|
3321
|
+
markerSha256,
|
|
3322
|
+
authorityId,
|
|
3323
|
+
authorityId,
|
|
3324
|
+
authorityId,
|
|
3325
|
+
authorityId,
|
|
3326
|
+
isolation,
|
|
3327
|
+
authorityId,
|
|
3328
|
+
authorityId,
|
|
3329
|
+
isolation
|
|
3330
|
+
],
|
|
3331
|
+
sql: `INSERT INTO oh_semantic_purges(authority_id, isolation_sha256,
|
|
3332
|
+
profile_sha256, published_generation, published_generation_sha256,
|
|
3333
|
+
purged_at, purge_marker_sha256, generation_count, membership_count,
|
|
3334
|
+
orphan_vector_count, isolation_scope_count, counts_recorded)
|
|
3335
|
+
SELECT ?, ?, ?,
|
|
3336
|
+
(SELECT generation FROM oh_semantic_heads WHERE authority_id = ?),
|
|
3337
|
+
(SELECT generation_sha256 FROM oh_semantic_heads WHERE authority_id = ?),
|
|
3338
|
+
?, ?,
|
|
3339
|
+
(SELECT count(*) FROM oh_semantic_generations WHERE authority_id = ?),
|
|
3340
|
+
(SELECT count(*) FROM oh_semantic_memberships WHERE authority_id = ?),
|
|
3341
|
+
(SELECT count(*) FROM oh_semantic_vectors AS vector
|
|
3342
|
+
JOIN oh_semantic_isolations AS isolation
|
|
3343
|
+
ON isolation.isolation_sha256 = vector.isolation_sha256
|
|
3344
|
+
WHERE isolation.authority_id = ?),
|
|
3345
|
+
(SELECT count(*) FROM oh_semantic_isolations WHERE authority_id = ?),
|
|
3346
|
+
1
|
|
3347
|
+
WHERE EXISTS (SELECT 1 FROM oh_semantic_isolations
|
|
3348
|
+
WHERE isolation_sha256 = ? AND authority_id = ?)
|
|
3349
|
+
AND NOT EXISTS (SELECT 1 FROM oh_semantic_heads
|
|
3350
|
+
WHERE authority_id = ? AND isolation_sha256 <> ?)
|
|
3351
|
+
ON CONFLICT DO NOTHING`
|
|
3352
|
+
},
|
|
3353
|
+
{
|
|
3354
|
+
args: [authorityId, authorityId, isolation],
|
|
3355
|
+
sql: `DELETE FROM oh_semantic_heads WHERE authority_id = ?
|
|
3356
|
+
AND EXISTS (SELECT 1 FROM oh_semantic_purges
|
|
3357
|
+
WHERE authority_id = ? AND isolation_sha256 = ?)`
|
|
3358
|
+
},
|
|
3359
|
+
{
|
|
3360
|
+
args: [authorityId, authorityId, isolation],
|
|
3361
|
+
sql: `DELETE FROM oh_semantic_memberships WHERE authority_id = ?
|
|
3362
|
+
AND EXISTS (SELECT 1 FROM oh_semantic_purges
|
|
3363
|
+
WHERE authority_id = ? AND isolation_sha256 = ?)`
|
|
3364
|
+
},
|
|
3365
|
+
{
|
|
3366
|
+
args: [authorityId, authorityId, isolation],
|
|
3367
|
+
sql: `DELETE FROM oh_semantic_vectors
|
|
3368
|
+
WHERE isolation_sha256 IN (SELECT isolation_sha256
|
|
3369
|
+
FROM oh_semantic_isolations WHERE authority_id = ?)
|
|
3370
|
+
AND EXISTS (SELECT 1 FROM oh_semantic_purges
|
|
3371
|
+
WHERE authority_id = ? AND isolation_sha256 = ?)`
|
|
3372
|
+
},
|
|
3373
|
+
{
|
|
3374
|
+
args: [authorityId, authorityId, isolation],
|
|
3375
|
+
sql: `DELETE FROM oh_semantic_generations WHERE authority_id = ?
|
|
3376
|
+
AND EXISTS (SELECT 1 FROM oh_semantic_purges
|
|
3377
|
+
WHERE authority_id = ? AND isolation_sha256 = ?)`
|
|
3378
|
+
}
|
|
3379
|
+
], "write");
|
|
3380
|
+
const receipt = await readPurge2(this.#client, authorityId);
|
|
3381
|
+
if (receipt === null) {
|
|
3382
|
+
throw new OhLibSqlSemanticV2Error("conflict", "The semantic purge identity changed before tombstoning.");
|
|
3383
|
+
}
|
|
3384
|
+
if (receipt.isolationSha256 !== isolation) {
|
|
3385
|
+
throw new OhLibSqlSemanticV2Error("conflict", "The semantic purge isolation conflicts.");
|
|
3386
|
+
}
|
|
3387
|
+
await verifyPurgeResidual(this.#client, authorityId);
|
|
3388
|
+
return receipt;
|
|
3389
|
+
}
|
|
3390
|
+
}
|
|
3391
|
+
async function openOhLibSqlSemanticCacheV2(client, options = {}) {
|
|
3392
|
+
return await OhLibSqlSemanticCacheV2.open(client, options.closeClient ?? false);
|
|
3393
|
+
}
|
|
1830
3394
|
export {
|
|
1831
3395
|
renderOhCloudflareEmbeddingQueryV1,
|
|
1832
3396
|
renderOhCloudflareEmbeddingDocumentV1,
|
|
3397
|
+
openOhLibSqlSemanticCacheV2,
|
|
1833
3398
|
openOhLibSqlSemanticCacheV1,
|
|
3399
|
+
deriveOhSemanticIsolationSha256V2,
|
|
3400
|
+
bootstrapOhLibSqlSemanticCacheV2,
|
|
1834
3401
|
bootstrapOhLibSqlSemanticCacheV1,
|
|
3402
|
+
OhLibSqlSemanticV2Error,
|
|
1835
3403
|
OhLibSqlSemanticError,
|
|
3404
|
+
OhLibSqlSemanticCacheV2,
|
|
1836
3405
|
OhLibSqlSemanticCacheV1,
|
|
1837
3406
|
OhCloudflareEmbeddingError,
|
|
1838
3407
|
OhCloudflareEmbeddingClientV1,
|
|
1839
3408
|
OH_SEMANTIC_RENDERER_V1,
|
|
3409
|
+
OH_LIBSQL_SEMANTIC_LIMITS_V2,
|
|
1840
3410
|
OH_LIBSQL_SEMANTIC_LIMITS_V1,
|
|
1841
3411
|
OH_CLOUDFLARE_EMBEDDING_PROFILE_V1,
|
|
1842
3412
|
OH_CLOUDFLARE_EMBEDDING_LIMITS_V1
|