@mnmal-ai/substrate 1.0.0-alpha.9 → 1.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -13
- package/dist/boot-store-BMQUbHKv.js +4086 -0
- package/dist/{runtime-metrics-DwpDJ8vz.d.ts → bridge-DnToJedN.d.ts} +2 -87
- package/dist/cli-DModIbnN.js +3390 -0
- package/dist/client.browser.js +2507 -0
- package/dist/client.d.ts +222 -171
- package/dist/client.js +6 -7
- package/dist/cluster-worker.js +0 -1
- package/dist/{content-type-C7T28xNb.js → content-type-wLljot5M.js} +0 -1
- package/dist/{env-utils-CGNazUNt.js → env-utils-s25Z1YSA.js} +0 -1
- package/dist/{errors-D4vKuwz8.d.ts → errors-Cmkp1qxf.d.ts} +44 -4
- package/dist/{fake-DUVtKVSE.js → fake-DdXtb-m8.js} +0 -1
- package/dist/{index-D7mSOFlM.d.ts → index-BI4KqFiR.d.ts} +7 -11
- package/dist/msgpackr-Dq18g6Ds.js +16 -0
- package/dist/node-version-BtK_Nq6z.js +32 -0
- package/dist/pack.d.ts +75 -11
- package/dist/pack.js +4 -4
- package/dist/{plugin-DpncES1p.js → plugin-CEOhrEme.js} +78 -8
- package/dist/proposals-bootstrap.js +6 -5
- package/dist/proposals.config.schema.json +5 -0
- package/dist/proposals.d.ts +17 -5
- package/dist/proposals.default-config.json +2 -1
- package/dist/proposals.js +2 -3
- package/dist/recall-bootstrap.js +3 -4
- package/dist/recall.d.ts +11 -12
- package/dist/recall.js +2 -3
- package/dist/{repl-DeKzfvNV.js → repl-BvDZy-Ut.js} +52 -26
- package/dist/{row-SRhura12.d.ts → row-D0If4qif.d.ts} +91 -13
- package/dist/run-BT123RV1.js +2443 -0
- package/dist/{plugin-t2g2wX0b.d.ts → schema-aIkY8sR3.d.ts} +1293 -63
- package/dist/{script-C6DSa_Na.js → script-ByQ9pzhC.js} +54 -26
- package/dist/script-runner.js +78 -0
- package/dist/sdk.d.ts +236 -38
- package/dist/sdk.js +5 -110
- package/dist/semver-D_zdq0hi.js +17 -0
- package/dist/src-BMD2OE--.js +8289 -0
- package/dist/{src-BLgbLq2k.js → src-B_kHkbsr.js} +780 -330
- package/dist/{src-DjIuRjzF.js → src-CKSJ13p8.js} +10231 -4237
- package/dist/{src-C12EA0w9.js → src-CW7eRDss.js} +179 -96
- package/dist/{row-DZ0Fnpc6.js → src-DmiUWOLp.js} +2 -1
- package/dist/{src-DCy7Iy2l.js → src-DmjwFMu8.js} +900 -77
- package/dist/{src-DiNTSaFk.js → src-FXO5JPyk.js} +686 -856
- package/dist/substrate-admin.js +17 -782
- package/dist/substrate.js +4 -5302
- package/dist/testing.d.ts +473 -96
- package/dist/testing.js +19 -165
- package/dist/{ws-C72iVS5T.d.ts → ws-DW8ng02H.d.ts} +4 -5
- package/package.json +26 -26
- package/dist/dataset-CqrqTYPz.js +0 -1535
- package/dist/pg-edges-store-KH_FQCxx.js +0 -381
- package/dist/schema-g4FQLr0V.d.ts +0 -433
- package/dist/src-BD27zeGF.js +0 -2339
|
@@ -1,9 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { B as ROW_FILTER_FIELD, V as matchesRowFilter, W as inFlightError, Y as unresolvedRow, Z as cosineSimilarity, et as memberKey } from "./plugin-DpncES1p.js";
|
|
1
|
+
import { i as txScopeIdFor, l as typeTableRef, s as guardedRefRead, u as whileNoTypeTable } from "./src-BMD2OE--.js";
|
|
2
|
+
import { B as ROW_FILTER_FIELD, V as matchesRowFilter, W as inFlightError, Y as unresolvedRow, Z as cosineSimilarity, et as memberKey } from "./plugin-CEOhrEme.js";
|
|
4
3
|
|
|
5
4
|
//#region ../../plugins/substrate-recall/package.json
|
|
6
|
-
var version = "1.0.0-
|
|
5
|
+
var version = "1.0.0-beta.1";
|
|
7
6
|
|
|
8
7
|
//#endregion
|
|
9
8
|
//#region ../../plugins/substrate-recall/src/in-memory-cluster-store.ts
|
|
@@ -184,7 +183,7 @@ var InMemoryEmbeddingsStore = class {
|
|
|
184
183
|
}
|
|
185
184
|
/**
|
|
186
185
|
* Materialize a row body via `datasetFor`, mirroring pg's join against
|
|
187
|
-
*
|
|
186
|
+
* the type's table. Returns UNRESOLVED when no dataset is wired, and `null`
|
|
188
187
|
* when the row is missing or deleted so the caller can exclude it.
|
|
189
188
|
*/
|
|
190
189
|
async #readRow(namespace, type, id) {
|
|
@@ -481,7 +480,7 @@ async function recountClusters(tx, clusterIds, stamps) {
|
|
|
481
480
|
WHERE c.id = counts.cluster_id
|
|
482
481
|
`;
|
|
483
482
|
}
|
|
484
|
-
/** Distinct from the migrator's `substrate-migrations` key;
|
|
483
|
+
/** Distinct from the migrator's `substrate-migrations` key; scoped by schema the same way. */
|
|
485
484
|
const FORMATION_LOCK_KEY = "substrate-recall-crystallise";
|
|
486
485
|
var PgClusterStore = class {
|
|
487
486
|
#sql;
|
|
@@ -492,7 +491,7 @@ var PgClusterStore = class {
|
|
|
492
491
|
}
|
|
493
492
|
/**
|
|
494
493
|
* Runs `fn` holding a session advisory lock that excludes formation in every
|
|
495
|
-
* process on this
|
|
494
|
+
* process on this schema; throws `crystallise_in_flight` if it is held.
|
|
496
495
|
*
|
|
497
496
|
* Lock and unlock share one reserved connection, since through the pool the
|
|
498
497
|
* unlock can land on another session and fail to release.
|
|
@@ -501,13 +500,13 @@ var PgClusterStore = class {
|
|
|
501
500
|
const conn = await this.#sql.reserve();
|
|
502
501
|
try {
|
|
503
502
|
const [row] = await conn`
|
|
504
|
-
SELECT pg_try_advisory_lock(hashtext(${FORMATION_LOCK_KEY})) AS acquired
|
|
503
|
+
SELECT pg_try_advisory_lock(hashtext(${FORMATION_LOCK_KEY} || ':' || current_schema())) AS acquired
|
|
505
504
|
`;
|
|
506
505
|
if (row?.acquired !== true) throw inFlightError();
|
|
507
506
|
try {
|
|
508
507
|
return await fn();
|
|
509
508
|
} finally {
|
|
510
|
-
await conn`SELECT pg_advisory_unlock(hashtext(${FORMATION_LOCK_KEY}))`;
|
|
509
|
+
await conn`SELECT pg_advisory_unlock(hashtext(${FORMATION_LOCK_KEY} || ':' || current_schema()))`;
|
|
511
510
|
}
|
|
512
511
|
} finally {
|
|
513
512
|
conn.release();
|
|
@@ -638,22 +637,56 @@ var PgClusterStore = class {
|
|
|
638
637
|
|
|
639
638
|
//#endregion
|
|
640
639
|
//#region ../../plugins/substrate-recall/src/pg-row-filter.ts
|
|
641
|
-
/**
|
|
640
|
+
/**
|
|
641
|
+
* SQL over the `r` alias shared by the pg recall stores: the relation `r`
|
|
642
|
+
* reads, the row body as the dataset read path returns it, and the row filter.
|
|
643
|
+
*
|
|
644
|
+
* `r` is a relation built from the type tables (`rowsFrom`), with the old
|
|
645
|
+
* `rows` table's columns (namespace, type, id, deleted_at, updated_at, body),
|
|
646
|
+
* so every statement reads it alike, and its `body` is the row `dataset.read`
|
|
647
|
+
* returns.
|
|
648
|
+
*/
|
|
649
|
+
/** The row body over `r`, as the dataset read path returns it: `rowsFrom` names it `body`. */
|
|
642
650
|
const ROW_BODY_SQL = "r.body";
|
|
651
|
+
/** {@link ScopedRef} of each `{ namespace, type }`. */
|
|
652
|
+
function scopedRefs(sql, scopes) {
|
|
653
|
+
return Promise.all(scopes.map(async ({ namespace, type }) => ({
|
|
654
|
+
namespace,
|
|
655
|
+
type,
|
|
656
|
+
ref: await typeTableRef(sql, namespace, type)
|
|
657
|
+
})));
|
|
658
|
+
}
|
|
659
|
+
const EMPTY_ROWS = "(SELECT NULL::text AS namespace, NULL::text AS type, NULL::text AS id, NULL::timestamptz AS deleted_at, NULL::timestamptz AS updated_at, NULL::jsonb AS body WHERE false)";
|
|
660
|
+
/**
|
|
661
|
+
* The relation the stores join as `r`: the scopes' type tables as one
|
|
662
|
+
* relation (namespace, type, id, deleted_at, updated_at, body). `wide` (one scope only) adds the
|
|
663
|
+
* type table's own columns, which `rowFilterSql` reads through `ref.text`.
|
|
664
|
+
*/
|
|
665
|
+
function rowsFrom(sql, scoped, wide = false) {
|
|
666
|
+
const present = scoped.flatMap((s) => s.ref === void 0 ? [] : [{
|
|
667
|
+
...s,
|
|
668
|
+
ref: s.ref
|
|
669
|
+
}]);
|
|
670
|
+
if (present.length === 0) return sql.unsafe(EMPTY_ROWS);
|
|
671
|
+
const columns = wide ? "x.*" : "x.id, x.deleted_at, x.updated_at";
|
|
672
|
+
let union = sql``;
|
|
673
|
+
for (const [i, { namespace, type, ref }] of present.entries()) union = sql`${union}${sql.unsafe(i === 0 ? "" : " UNION ALL ")}SELECT ${namespace}::text AS namespace, ${type}::text AS type, ${sql.unsafe(`${columns}, ${ref.rowJson("x")} AS body FROM ${ref.from} x`)}`;
|
|
674
|
+
return sql`(${union})`;
|
|
675
|
+
}
|
|
643
676
|
/**
|
|
644
677
|
* The row filter as `AND …` clauses, for appending to a WHERE. Every
|
|
645
|
-
* comparison is as text against the
|
|
646
|
-
* `matchesRowFilter
|
|
647
|
-
* against ROW_FILTER_FIELD and values are bound, so nothing
|
|
648
|
-
* spliced into SQL.
|
|
678
|
+
* comparison is as text against the row, the same semantics as
|
|
679
|
+
* `matchesRowFilter`: `ref.text`, over the columns a `wide` `rowsFrom` exposes. Field names
|
|
680
|
+
* are re-checked against ROW_FILTER_FIELD and values are bound, so nothing
|
|
681
|
+
* caller-supplied is spliced into SQL unquoted.
|
|
649
682
|
*/
|
|
650
|
-
function rowFilterSql(tx, filter) {
|
|
683
|
+
function rowFilterSql(tx, ref, filter) {
|
|
651
684
|
if (filter === void 0) return tx``;
|
|
652
685
|
let out = tx``;
|
|
653
686
|
for (const [field, pred] of Object.entries(filter)) {
|
|
654
687
|
if (!ROW_FILTER_FIELD.test(field)) throw new Error(`row filter field '${field}' is not a plain identifier`);
|
|
655
|
-
const text = tx
|
|
656
|
-
const isNull = tx
|
|
688
|
+
const text = tx.unsafe(ref.text("r", field));
|
|
689
|
+
const isNull = tx.unsafe(`${ref.text("r", field)} IS NULL`);
|
|
657
690
|
const values = pred.in ?? [pred.eq ?? null];
|
|
658
691
|
const strings = values.filter((v) => v !== null).map((v) => String(v));
|
|
659
692
|
const wantsNull = values.some((v) => v === null);
|
|
@@ -662,6 +695,37 @@ function rowFilterSql(tx, filter) {
|
|
|
662
695
|
}
|
|
663
696
|
return out;
|
|
664
697
|
}
|
|
698
|
+
/**
|
|
699
|
+
* `purgeOrphans` on `tables`: once per distinct type in `table` (`embeddings`
|
|
700
|
+
* or `lexical`), deletes the entries whose row is missing or tombstoned in its
|
|
701
|
+
* type table. A type with no table deletes all of its entries.
|
|
702
|
+
*
|
|
703
|
+
* @returns the number of entries deleted.
|
|
704
|
+
*/
|
|
705
|
+
async function purgeTypeTableOrphans(sql, table) {
|
|
706
|
+
const types = await sql`
|
|
707
|
+
SELECT DISTINCT row_namespace AS namespace, row_type AS type FROM ${sql(table)}`;
|
|
708
|
+
let deleted = 0;
|
|
709
|
+
for (const scope of await scopedRefs(sql, types)) {
|
|
710
|
+
if (scope.ref === void 0) {
|
|
711
|
+
const result = await whileNoTypeTable(sql, scope.namespace, scope.type, (tx) => tx`DELETE FROM ${tx(table)}
|
|
712
|
+
WHERE row_namespace = ${scope.namespace} AND row_type = ${scope.type}`);
|
|
713
|
+
deleted += result?.count ?? 0;
|
|
714
|
+
continue;
|
|
715
|
+
}
|
|
716
|
+
const result = await sql`
|
|
717
|
+
DELETE FROM ${sql(table)} o
|
|
718
|
+
WHERE o.row_namespace = ${scope.namespace}
|
|
719
|
+
AND o.row_type = ${scope.type}
|
|
720
|
+
AND NOT EXISTS (
|
|
721
|
+
SELECT 1 FROM ${rowsFrom(sql, [scope])} r
|
|
722
|
+
WHERE r.id = o.row_id AND r.deleted_at IS NULL
|
|
723
|
+
)
|
|
724
|
+
`;
|
|
725
|
+
deleted += result.count;
|
|
726
|
+
}
|
|
727
|
+
return deleted;
|
|
728
|
+
}
|
|
665
729
|
|
|
666
730
|
//#endregion
|
|
667
731
|
//#region ../../plugins/substrate-recall/src/pg-embeddings-store.ts
|
|
@@ -735,17 +799,22 @@ var PgEmbeddingsStore = class {
|
|
|
735
799
|
const threshold = typeof input.threshold === "number" ? input.threshold : 0;
|
|
736
800
|
const wantVector = input.returnVector === true;
|
|
737
801
|
const halfLife = input.salienceHalfLifeHours;
|
|
738
|
-
return (await sql.begin(async (tx) => {
|
|
802
|
+
return (await guardedRefRead(sql, input.namespace, input.type, (ref, guard) => sql.begin(async (tx) => {
|
|
803
|
+
const scoped = [{
|
|
804
|
+
namespace: input.namespace,
|
|
805
|
+
type: input.type,
|
|
806
|
+
ref
|
|
807
|
+
}];
|
|
739
808
|
await tx`SET LOCAL hnsw.iterative_scan = 'strict_order'`;
|
|
740
809
|
await tx.unsafe(`SET LOCAL hnsw.ef_search = ${DEFAULT_HNSW_EF_SEARCH}`);
|
|
741
|
-
return tx`
|
|
810
|
+
return guard(tx, tx`
|
|
742
811
|
SELECT
|
|
743
812
|
${tx.unsafe(ROW_BODY_SQL)} AS row,
|
|
744
813
|
1 - (e.embedding <=> ${literal}::vector) AS score,
|
|
745
814
|
e.salience ${halfLife !== void 0 && halfLife > 0 ? tx`* exp(-ln(2) * GREATEST(0, EXTRACT(EPOCH FROM (now() - e.salience_updated_at))) / ${halfLife * 3600})` : tx``} AS salience
|
|
746
815
|
${wantVector ? tx`, e.embedding::text AS embedding` : tx``}
|
|
747
816
|
FROM embeddings e
|
|
748
|
-
JOIN
|
|
817
|
+
JOIN ${rowsFrom(tx, scoped, true)} r
|
|
749
818
|
ON r.namespace = e.row_namespace
|
|
750
819
|
AND r.type = e.row_type
|
|
751
820
|
AND r.id = e.row_id
|
|
@@ -754,11 +823,11 @@ var PgEmbeddingsStore = class {
|
|
|
754
823
|
AND e.model = ${input.model}
|
|
755
824
|
AND r.deleted_at IS NULL
|
|
756
825
|
AND 1 - (e.embedding <=> ${literal}::vector) >= ${threshold}
|
|
757
|
-
${rowFilterSql(tx, input.rowFilter)}
|
|
826
|
+
${rowFilterSql(tx, ref, input.rowFilter)}
|
|
758
827
|
ORDER BY e.embedding <=> ${literal}::vector
|
|
759
828
|
LIMIT ${k}
|
|
760
|
-
|
|
761
|
-
})).map((r) => ({
|
|
829
|
+
`);
|
|
830
|
+
}), "row")).map((r) => ({
|
|
762
831
|
row: r.row,
|
|
763
832
|
score: r.score,
|
|
764
833
|
salience: r.salience,
|
|
@@ -795,21 +864,28 @@ var PgEmbeddingsStore = class {
|
|
|
795
864
|
async fetchEmbeddings(ids, model) {
|
|
796
865
|
if (ids.length === 0) return [];
|
|
797
866
|
const sql = this.#sql;
|
|
798
|
-
|
|
867
|
+
const byType = /* @__PURE__ */ new Map();
|
|
868
|
+
for (const id of ids) {
|
|
869
|
+
const key = `${id.namespace}/${id.type}`;
|
|
870
|
+
byType.set(key, [...byType.get(key) ?? [], id]);
|
|
871
|
+
}
|
|
872
|
+
const scoped = await scopedRefs(sql, [...byType.values()].map(([first]) => first));
|
|
873
|
+
const fetch = (part, from) => sql`
|
|
799
874
|
SELECT e.row_namespace, e.row_type, e.row_id, e.embedding::text AS embedding,
|
|
800
875
|
${sql.unsafe(ROW_BODY_SQL)} AS body, e.salience
|
|
801
|
-
FROM (VALUES ${sql(
|
|
876
|
+
FROM (VALUES ${sql(part.map((id) => [
|
|
802
877
|
id.namespace,
|
|
803
878
|
id.type,
|
|
804
879
|
id.id
|
|
805
880
|
]))}) AS t(ns, ty, id)
|
|
806
881
|
JOIN embeddings e
|
|
807
882
|
ON e.row_namespace = t.ns AND e.row_type = t.ty AND e.row_id = t.id
|
|
808
|
-
JOIN
|
|
883
|
+
JOIN ${from} r
|
|
809
884
|
ON r.namespace = e.row_namespace AND r.type = e.row_type AND r.id = e.row_id
|
|
810
885
|
WHERE e.model = ${model}
|
|
811
886
|
AND r.deleted_at IS NULL
|
|
812
|
-
|
|
887
|
+
`;
|
|
888
|
+
return (await Promise.all(scoped.filter((s) => s.ref !== void 0).map((s) => fetch(byType.get(`${s.namespace}/${s.type}`) ?? [], rowsFrom(sql, [s]))))).flat().map((r) => ({
|
|
813
889
|
namespace: r.row_namespace,
|
|
814
890
|
type: r.row_type,
|
|
815
891
|
id: r.row_id,
|
|
@@ -820,40 +896,39 @@ var PgEmbeddingsStore = class {
|
|
|
820
896
|
}
|
|
821
897
|
async selectCandidates(selector, model) {
|
|
822
898
|
const sql = this.#sql;
|
|
823
|
-
const
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
899
|
+
const arms = (await scopedRefs(sql, selector.scopes)).flatMap((s) => s.ref === void 0 ? [] : [{
|
|
900
|
+
...s,
|
|
901
|
+
ref: s.ref
|
|
902
|
+
}]);
|
|
903
|
+
if (arms.length === 0) return [];
|
|
904
|
+
let kept = sql``;
|
|
905
|
+
for (const [i, { namespace, type, ref }] of arms.entries()) {
|
|
906
|
+
let keep = sql`x.deleted_at IS NULL`;
|
|
907
|
+
for (const { field, prefix } of selector.excludeTextPrefixes) keep = sql`${keep} AND NOT COALESCE(
|
|
908
|
+
jsonb_typeof(${sql.unsafe(ref.json("x", field))}) IN ('string', 'number', 'boolean')
|
|
909
|
+
AND starts_with(${sql.unsafe(ref.text("x", field))}, ${prefix}::text), false)`;
|
|
910
|
+
kept = sql`${kept}${sql.unsafe(i === 0 ? "" : " UNION ALL ")}SELECT ${namespace}::text AS namespace, ${type}::text AS type, x.id FROM ${sql.unsafe(ref.from)} x WHERE ${keep}`;
|
|
911
|
+
}
|
|
827
912
|
return (await sql`
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
-- to scalar JSON values (jsonb_typeof) to match the in-memory side,
|
|
848
|
-
-- which coerces only string/number/boolean via String(value) — a
|
|
849
|
-
-- composite (object/array) has no agreed-on text rendering between
|
|
850
|
-
-- pg's jsonb ->> and JS's String(), so both sides now treat a
|
|
851
|
-
-- composite as never-excluded rather than risk disagreeing.
|
|
852
|
-
WHERE jsonb_typeof(${sql.unsafe(ROW_BODY_SQL)} -> ex.field) IN ('string', 'number', 'boolean')
|
|
853
|
-
AND starts_with(${sql.unsafe(ROW_BODY_SQL)} ->> ex.field, ex.prefix)
|
|
854
|
-
)
|
|
855
|
-
ORDER BY e.row_namespace COLLATE "C" ASC, e.row_type COLLATE "C" ASC, e.row_id COLLATE "C" ASC
|
|
856
|
-
LIMIT ${selector.limit}
|
|
913
|
+
WITH picked AS (
|
|
914
|
+
SELECT e.row_namespace, e.row_type, e.row_id
|
|
915
|
+
FROM embeddings e
|
|
916
|
+
JOIN (${kept}) k
|
|
917
|
+
ON k.namespace = e.row_namespace AND k.type = e.row_type AND k.id = e.row_id
|
|
918
|
+
WHERE e.model = ${model}
|
|
919
|
+
ORDER BY e.row_namespace COLLATE "C" ASC, e.row_type COLLATE "C" ASC, e.row_id COLLATE "C" ASC
|
|
920
|
+
LIMIT ${selector.limit}
|
|
921
|
+
)
|
|
922
|
+
SELECT p.row_namespace, p.row_type, p.row_id, e.embedding::text AS embedding, b.body
|
|
923
|
+
FROM picked p
|
|
924
|
+
JOIN embeddings e
|
|
925
|
+
ON e.row_namespace = p.row_namespace AND e.row_type = p.row_type AND e.row_id = p.row_id
|
|
926
|
+
CROSS JOIN LATERAL (
|
|
927
|
+
SELECT ${sql.unsafe(ROW_BODY_SQL)} AS body FROM ${rowsFrom(sql, arms)} r
|
|
928
|
+
WHERE r.namespace = p.row_namespace AND r.type = p.row_type AND r.id = p.row_id
|
|
929
|
+
OFFSET 0
|
|
930
|
+
) b
|
|
931
|
+
ORDER BY p.row_namespace COLLATE "C" ASC, p.row_type COLLATE "C" ASC, p.row_id COLLATE "C" ASC
|
|
857
932
|
`).map((r) => ({
|
|
858
933
|
namespace: r.row_namespace,
|
|
859
934
|
type: r.row_type,
|
|
@@ -862,17 +937,16 @@ var PgEmbeddingsStore = class {
|
|
|
862
937
|
row: r.body
|
|
863
938
|
}));
|
|
864
939
|
}
|
|
865
|
-
/**
|
|
866
|
-
* Deletes every embedding whose backing row is missing or soft-deleted.
|
|
867
|
-
* Needed because `embeddings` has no FK to `rows` and no
|
|
868
|
-
* dataset delete path touches it; tombstones are terminal, so soft-deleted
|
|
869
|
-
* rows are safe to collect immediately. Run as a sweep stage by the binary.
|
|
870
|
-
*
|
|
871
|
-
* @returns number of orphaned embeddings deleted.
|
|
872
|
-
*/
|
|
873
940
|
async staleRowIds(input) {
|
|
874
|
-
|
|
875
|
-
|
|
941
|
+
const sql = this.#sql;
|
|
942
|
+
const ref = await typeTableRef(sql, input.namespace, input.type);
|
|
943
|
+
if (ref === void 0) return [];
|
|
944
|
+
return (await sql`
|
|
945
|
+
SELECT r.id FROM ${rowsFrom(sql, [{
|
|
946
|
+
namespace: input.namespace,
|
|
947
|
+
type: input.type,
|
|
948
|
+
ref
|
|
949
|
+
}])} r
|
|
876
950
|
WHERE r.namespace = ${input.namespace}
|
|
877
951
|
AND r.type = ${input.type}
|
|
878
952
|
AND r.deleted_at IS NULL
|
|
@@ -888,22 +962,29 @@ var PgEmbeddingsStore = class {
|
|
|
888
962
|
LIMIT ${input.limit}
|
|
889
963
|
`).map((r) => r.id);
|
|
890
964
|
}
|
|
965
|
+
/**
|
|
966
|
+
* Deletes every embedding whose backing row is missing or soft-deleted.
|
|
967
|
+
* Needed because `embeddings` has no FK to the type tables and no
|
|
968
|
+
* dataset delete path touches it; tombstones are terminal, so soft-deleted
|
|
969
|
+
* rows are safe to collect immediately. Run as a sweep stage by the binary.
|
|
970
|
+
*
|
|
971
|
+
* @returns number of orphaned embeddings deleted.
|
|
972
|
+
*/
|
|
891
973
|
async purgeOrphans() {
|
|
892
|
-
return (
|
|
893
|
-
DELETE FROM embeddings e
|
|
894
|
-
WHERE NOT EXISTS (
|
|
895
|
-
SELECT 1 FROM rows r
|
|
896
|
-
WHERE r.namespace = e.row_namespace
|
|
897
|
-
AND r.type = e.row_type
|
|
898
|
-
AND r.id = e.row_id
|
|
899
|
-
AND r.deleted_at IS NULL
|
|
900
|
-
)
|
|
901
|
-
`).count;
|
|
974
|
+
return purgeTypeTableOrphans(this.#sql, "embeddings");
|
|
902
975
|
}
|
|
903
976
|
};
|
|
904
977
|
|
|
905
978
|
//#endregion
|
|
906
979
|
//#region ../../plugins/substrate-recall/src/pg-lexical-store.ts
|
|
980
|
+
/**
|
|
981
|
+
* Postgres-backed `LexicalStore` — `lexical` + `ts_rank` over its GIN
|
|
982
|
+
* index. Counterpart to `PgEmbeddingsStore`.
|
|
983
|
+
*
|
|
984
|
+
* Search joins the type's rows the same way the KNN path does, with the same
|
|
985
|
+
* `deleted_at IS NULL` predicate, so a deleted row cannot surface through this
|
|
986
|
+
* arm after it has stopped surfacing through the dense one.
|
|
987
|
+
*/
|
|
907
988
|
/** Hard ceiling on `k`, mirroring the embeddings store's cap. */
|
|
908
989
|
const MAX_K = 100;
|
|
909
990
|
var PgLexicalStore = class {
|
|
@@ -933,27 +1014,24 @@ var PgLexicalStore = class {
|
|
|
933
1014
|
/**
|
|
934
1015
|
* Deletes every lexical row whose backing row is missing or soft-deleted,
|
|
935
1016
|
* as `PgEmbeddingsStore.purgeOrphans` does for vectors: `lexical` has no FK
|
|
936
|
-
* to
|
|
1017
|
+
* to the type tables, so a drop, purge or tombstone would otherwise leave the text.
|
|
937
1018
|
*
|
|
938
1019
|
* @returns number of orphaned lexical rows deleted.
|
|
939
1020
|
*/
|
|
940
1021
|
async purgeOrphans() {
|
|
941
|
-
return (
|
|
942
|
-
DELETE FROM lexical l
|
|
943
|
-
WHERE NOT EXISTS (
|
|
944
|
-
SELECT 1 FROM rows r
|
|
945
|
-
WHERE r.namespace = l.row_namespace
|
|
946
|
-
AND r.type = l.row_type
|
|
947
|
-
AND r.id = l.row_id
|
|
948
|
-
AND r.deleted_at IS NULL
|
|
949
|
-
)
|
|
950
|
-
`).count;
|
|
1022
|
+
return purgeTypeTableOrphans(this.#sql, "lexical");
|
|
951
1023
|
}
|
|
952
1024
|
async countIndexed(namespace, type) {
|
|
1025
|
+
const ref = await typeTableRef(this.#sql, namespace, type);
|
|
1026
|
+
if (ref === void 0) return 0;
|
|
953
1027
|
const rows = await this.#sql`
|
|
954
1028
|
SELECT COUNT(*) AS n
|
|
955
1029
|
FROM lexical l
|
|
956
|
-
JOIN
|
|
1030
|
+
JOIN ${rowsFrom(this.#sql, [{
|
|
1031
|
+
namespace,
|
|
1032
|
+
type,
|
|
1033
|
+
ref
|
|
1034
|
+
}])} r
|
|
957
1035
|
ON r.namespace = l.row_namespace
|
|
958
1036
|
AND r.type = l.row_type
|
|
959
1037
|
AND r.id = l.row_id
|
|
@@ -979,6 +1057,7 @@ var PgLexicalStore = class {
|
|
|
979
1057
|
if (strict.length > 0) return strict;
|
|
980
1058
|
return this.#run(input, k, true);
|
|
981
1059
|
}
|
|
1060
|
+
/** One pass, under the type table's read guard, so a promotion meanwhile cannot drop a field or a hit. */
|
|
982
1061
|
async #run(input, k, degrade) {
|
|
983
1062
|
const query = degrade ? this.#sql`(
|
|
984
1063
|
SELECT CASE
|
|
@@ -987,7 +1066,7 @@ var PgLexicalStore = class {
|
|
|
987
1066
|
END AS query
|
|
988
1067
|
FROM (SELECT websearch_to_tsquery(${this.#tsConfig}, ${input.query})::text AS t) s
|
|
989
1068
|
)` : this.#sql`websearch_to_tsquery(${this.#tsConfig}, ${input.query})`;
|
|
990
|
-
return (await this.#sql`
|
|
1069
|
+
return (await guardedRefRead(this.#sql, input.namespace, input.type, (ref, guard) => guard(this.#sql, this.#sql`
|
|
991
1070
|
SELECT
|
|
992
1071
|
l.row_namespace,
|
|
993
1072
|
l.row_type,
|
|
@@ -996,7 +1075,11 @@ var PgLexicalStore = class {
|
|
|
996
1075
|
ts_rank(l.tsv, q.query) AS score
|
|
997
1076
|
FROM lexical l
|
|
998
1077
|
CROSS JOIN ${query} AS q(query)
|
|
999
|
-
JOIN
|
|
1078
|
+
JOIN ${rowsFrom(this.#sql, [{
|
|
1079
|
+
namespace: input.namespace,
|
|
1080
|
+
type: input.type,
|
|
1081
|
+
ref
|
|
1082
|
+
}], true)} r
|
|
1000
1083
|
ON r.namespace = l.row_namespace
|
|
1001
1084
|
AND r.type = l.row_type
|
|
1002
1085
|
AND r.id = l.row_id
|
|
@@ -1005,10 +1088,10 @@ var PgLexicalStore = class {
|
|
|
1005
1088
|
AND r.deleted_at IS NULL
|
|
1006
1089
|
AND q.query IS NOT NULL
|
|
1007
1090
|
AND l.tsv @@ q.query
|
|
1008
|
-
${rowFilterSql(this.#sql, input.rowFilter)}
|
|
1091
|
+
${rowFilterSql(this.#sql, ref, input.rowFilter)}
|
|
1009
1092
|
ORDER BY score DESC, l.row_id ASC
|
|
1010
1093
|
LIMIT ${k}
|
|
1011
|
-
`).map((r) => ({
|
|
1094
|
+
`), "row_id")).map((r) => ({
|
|
1012
1095
|
rowNamespace: r.row_namespace,
|
|
1013
1096
|
rowType: r.row_type,
|
|
1014
1097
|
rowId: r.row_id,
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import
|
|
1
|
+
import "./src-B_kHkbsr.js";
|
|
2
|
+
|
|
2
3
|
//#region ../../packages/substrate-policy/src/encoding.ts
|
|
3
4
|
/**
|
|
4
5
|
* Comma-separated string-list encoding. Spec §5.12.
|