@mnmal-ai/substrate 1.0.0-alpha.10 → 1.0.0-alpha.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/{boot-store-BVqfFxh1.js → boot-store-DHv9H1Gr.js} +170 -69
- package/dist/{bridge-Cp4PaIYO.d.ts → bridge-DQyQ6Qhn.d.ts} +1 -1
- package/dist/{cli-hSCBywhE.js → cli-_UNdneXr.js} +44 -24
- package/dist/client.d.ts +3 -4
- package/dist/client.js +3 -3
- package/dist/{errors-7mycvNGq.d.ts → errors-xP9Mo7Q1.d.ts} +38 -4
- package/dist/{index-DgSfwx7M.d.ts → index-baF5A5jU.d.ts} +1 -1
- package/dist/pack.d.ts +2 -2
- package/dist/pack.js +1 -1
- package/dist/{pg-edges-store-Lxsl5ZnK.js → pg-edges-store-B4FSN9yp.js} +72 -9
- package/dist/{plugin-CtwOINCM.js → plugin-BLm5IRdX.js} +2 -2
- package/dist/proposals-bootstrap.js +2 -2
- package/dist/proposals.d.ts +2 -3
- package/dist/proposals.js +1 -1
- package/dist/recall-bootstrap.js +2 -2
- package/dist/recall.d.ts +10 -11
- package/dist/recall.js +2 -2
- package/dist/{repl-DqfHelK7.js → repl-DUJGBwie.js} +25 -25
- package/dist/{row-BKKCXIG6.d.ts → row-Cu_iRhcR.d.ts} +57 -10
- package/dist/{run-B5dTbZ3g.js → run-BpsTNvdI.js} +539 -145
- package/dist/{dataset-CejF-Fr_.d.ts → schema-oXfQyo0s.d.ts} +2455 -1589
- package/dist/sdk.d.ts +108 -34
- package/dist/sdk.js +4 -4
- package/dist/{src-CqT-QWZ_.js → src-Bew_JvTo.js} +32 -16
- package/dist/{src-Da0t6si5.js → src-Bi3NhEsT.js} +1 -1
- package/dist/{src-5SJ5GEIM.js → src-C4t1Gy-m.js} +4844 -1367
- package/dist/{src-7CUSMK66.js → src-CHND7-Pm.js} +7 -3
- package/dist/src-CSe2tJDf.js +6940 -0
- package/dist/{src-Bi849yhn.js → src-CzhKLsOV.js} +175 -91
- package/dist/{src-hZb8wfvg.js → src-DWqzrm3O.js} +400 -284
- package/dist/substrate-admin.js +1 -1
- package/dist/substrate.js +1 -1
- package/dist/testing.d.ts +255 -90
- package/dist/testing.js +6 -23
- package/dist/{ws-ByZyHk7m.d.ts → ws-Dx3A2B4U.d.ts} +2 -3
- package/package.json +20 -20
- package/dist/schema-DoSc-4ig.d.ts +0 -470
- package/dist/src-CyvV7dCK.js +0 -4333
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
|
|
2
|
-
import { n as txScopeIdFor } from "./src-
|
|
3
|
-
import { B as ROW_FILTER_FIELD, V as matchesRowFilter, W as inFlightError, Y as unresolvedRow, Z as cosineSimilarity, et as memberKey } from "./plugin-
|
|
2
|
+
import { a as guardedRefRead, c as whileNoTypeTable, n as txScopeIdFor, s as typeTableRef } from "./src-CSe2tJDf.js";
|
|
3
|
+
import { B as ROW_FILTER_FIELD, V as matchesRowFilter, W as inFlightError, Y as unresolvedRow, Z as cosineSimilarity, et as memberKey } from "./plugin-BLm5IRdX.js";
|
|
4
4
|
|
|
5
5
|
//#region ../../plugins/substrate-recall/package.json
|
|
6
|
-
var version = "1.0.0-alpha.
|
|
6
|
+
var version = "1.0.0-alpha.12";
|
|
7
7
|
|
|
8
8
|
//#endregion
|
|
9
9
|
//#region ../../plugins/substrate-recall/src/in-memory-cluster-store.ts
|
|
@@ -184,7 +184,7 @@ var InMemoryEmbeddingsStore = class {
|
|
|
184
184
|
}
|
|
185
185
|
/**
|
|
186
186
|
* Materialize a row body via `datasetFor`, mirroring pg's join against
|
|
187
|
-
*
|
|
187
|
+
* the type's table. Returns UNRESOLVED when no dataset is wired, and `null`
|
|
188
188
|
* when the row is missing or deleted so the caller can exclude it.
|
|
189
189
|
*/
|
|
190
190
|
async #readRow(namespace, type, id) {
|
|
@@ -638,22 +638,56 @@ var PgClusterStore = class {
|
|
|
638
638
|
|
|
639
639
|
//#endregion
|
|
640
640
|
//#region ../../plugins/substrate-recall/src/pg-row-filter.ts
|
|
641
|
-
/**
|
|
641
|
+
/**
|
|
642
|
+
* SQL over the `r` alias shared by the pg recall stores: the relation `r`
|
|
643
|
+
* reads, the row body as the dataset read path returns it, and the row filter.
|
|
644
|
+
*
|
|
645
|
+
* `r` is a relation built from the type tables (`rowsFrom`), with the old
|
|
646
|
+
* `rows` table's columns (namespace, type, id, deleted_at, updated_at, body),
|
|
647
|
+
* so every statement reads it alike, and its `body` is the row `dataset.read`
|
|
648
|
+
* returns.
|
|
649
|
+
*/
|
|
650
|
+
/** The row body over `r`, as the dataset read path returns it: `rowsFrom` names it `body`. */
|
|
642
651
|
const ROW_BODY_SQL = "r.body";
|
|
652
|
+
/** {@link ScopedRef} of each `{ namespace, type }`. */
|
|
653
|
+
function scopedRefs(sql, scopes) {
|
|
654
|
+
return Promise.all(scopes.map(async ({ namespace, type }) => ({
|
|
655
|
+
namespace,
|
|
656
|
+
type,
|
|
657
|
+
ref: await typeTableRef(sql, namespace, type)
|
|
658
|
+
})));
|
|
659
|
+
}
|
|
660
|
+
const EMPTY_ROWS = "(SELECT NULL::text AS namespace, NULL::text AS type, NULL::text AS id, NULL::timestamptz AS deleted_at, NULL::timestamptz AS updated_at, NULL::jsonb AS body WHERE false)";
|
|
661
|
+
/**
|
|
662
|
+
* The relation the stores join as `r`: the scopes' type tables as one
|
|
663
|
+
* relation (namespace, type, id, deleted_at, updated_at, body). `wide` (one scope only) adds the
|
|
664
|
+
* type table's own columns, which `rowFilterSql` reads through `ref.text`.
|
|
665
|
+
*/
|
|
666
|
+
function rowsFrom(sql, scoped, wide = false) {
|
|
667
|
+
const present = scoped.flatMap((s) => s.ref === void 0 ? [] : [{
|
|
668
|
+
...s,
|
|
669
|
+
ref: s.ref
|
|
670
|
+
}]);
|
|
671
|
+
if (present.length === 0) return sql.unsafe(EMPTY_ROWS);
|
|
672
|
+
const columns = wide ? "x.*" : "x.id, x.deleted_at, x.updated_at";
|
|
673
|
+
let union = sql``;
|
|
674
|
+
for (const [i, { namespace, type, ref }] of present.entries()) union = sql`${union}${sql.unsafe(i === 0 ? "" : " UNION ALL ")}SELECT ${namespace}::text AS namespace, ${type}::text AS type, ${sql.unsafe(`${columns}, ${ref.rowJson("x")} AS body FROM ${ref.from} x`)}`;
|
|
675
|
+
return sql`(${union})`;
|
|
676
|
+
}
|
|
643
677
|
/**
|
|
644
678
|
* The row filter as `AND …` clauses, for appending to a WHERE. Every
|
|
645
|
-
* comparison is as text against the
|
|
646
|
-
* `matchesRowFilter
|
|
647
|
-
* against ROW_FILTER_FIELD and values are bound, so nothing
|
|
648
|
-
* spliced into SQL.
|
|
679
|
+
* comparison is as text against the row, the same semantics as
|
|
680
|
+
* `matchesRowFilter`: `ref.text`, over the columns a `wide` `rowsFrom` exposes. Field names
|
|
681
|
+
* are re-checked against ROW_FILTER_FIELD and values are bound, so nothing
|
|
682
|
+
* caller-supplied is spliced into SQL unquoted.
|
|
649
683
|
*/
|
|
650
|
-
function rowFilterSql(tx, filter) {
|
|
684
|
+
function rowFilterSql(tx, ref, filter) {
|
|
651
685
|
if (filter === void 0) return tx``;
|
|
652
686
|
let out = tx``;
|
|
653
687
|
for (const [field, pred] of Object.entries(filter)) {
|
|
654
688
|
if (!ROW_FILTER_FIELD.test(field)) throw new Error(`row filter field '${field}' is not a plain identifier`);
|
|
655
|
-
const text = tx
|
|
656
|
-
const isNull = tx
|
|
689
|
+
const text = tx.unsafe(ref.text("r", field));
|
|
690
|
+
const isNull = tx.unsafe(`${ref.text("r", field)} IS NULL`);
|
|
657
691
|
const values = pred.in ?? [pred.eq ?? null];
|
|
658
692
|
const strings = values.filter((v) => v !== null).map((v) => String(v));
|
|
659
693
|
const wantsNull = values.some((v) => v === null);
|
|
@@ -662,6 +696,37 @@ function rowFilterSql(tx, filter) {
|
|
|
662
696
|
}
|
|
663
697
|
return out;
|
|
664
698
|
}
|
|
699
|
+
/**
|
|
700
|
+
* `purgeOrphans` on `tables`: once per distinct type in `table` (`embeddings`
|
|
701
|
+
* or `lexical`), deletes the entries whose row is missing or tombstoned in its
|
|
702
|
+
* type table. A type with no table deletes all of its entries.
|
|
703
|
+
*
|
|
704
|
+
* @returns the number of entries deleted.
|
|
705
|
+
*/
|
|
706
|
+
async function purgeTypeTableOrphans(sql, table) {
|
|
707
|
+
const types = await sql`
|
|
708
|
+
SELECT DISTINCT row_namespace AS namespace, row_type AS type FROM ${sql(table)}`;
|
|
709
|
+
let deleted = 0;
|
|
710
|
+
for (const scope of await scopedRefs(sql, types)) {
|
|
711
|
+
if (scope.ref === void 0) {
|
|
712
|
+
const result = await whileNoTypeTable(sql, scope.namespace, scope.type, (tx) => tx`DELETE FROM ${tx(table)}
|
|
713
|
+
WHERE row_namespace = ${scope.namespace} AND row_type = ${scope.type}`);
|
|
714
|
+
deleted += result?.count ?? 0;
|
|
715
|
+
continue;
|
|
716
|
+
}
|
|
717
|
+
const result = await sql`
|
|
718
|
+
DELETE FROM ${sql(table)} o
|
|
719
|
+
WHERE o.row_namespace = ${scope.namespace}
|
|
720
|
+
AND o.row_type = ${scope.type}
|
|
721
|
+
AND NOT EXISTS (
|
|
722
|
+
SELECT 1 FROM ${rowsFrom(sql, [scope])} r
|
|
723
|
+
WHERE r.id = o.row_id AND r.deleted_at IS NULL
|
|
724
|
+
)
|
|
725
|
+
`;
|
|
726
|
+
deleted += result.count;
|
|
727
|
+
}
|
|
728
|
+
return deleted;
|
|
729
|
+
}
|
|
665
730
|
|
|
666
731
|
//#endregion
|
|
667
732
|
//#region ../../plugins/substrate-recall/src/pg-embeddings-store.ts
|
|
@@ -735,17 +800,22 @@ var PgEmbeddingsStore = class {
|
|
|
735
800
|
const threshold = typeof input.threshold === "number" ? input.threshold : 0;
|
|
736
801
|
const wantVector = input.returnVector === true;
|
|
737
802
|
const halfLife = input.salienceHalfLifeHours;
|
|
738
|
-
return (await sql.begin(async (tx) => {
|
|
803
|
+
return (await guardedRefRead(sql, input.namespace, input.type, (ref, guard) => sql.begin(async (tx) => {
|
|
804
|
+
const scoped = [{
|
|
805
|
+
namespace: input.namespace,
|
|
806
|
+
type: input.type,
|
|
807
|
+
ref
|
|
808
|
+
}];
|
|
739
809
|
await tx`SET LOCAL hnsw.iterative_scan = 'strict_order'`;
|
|
740
810
|
await tx.unsafe(`SET LOCAL hnsw.ef_search = ${DEFAULT_HNSW_EF_SEARCH}`);
|
|
741
|
-
return tx`
|
|
811
|
+
return guard(tx, tx`
|
|
742
812
|
SELECT
|
|
743
813
|
${tx.unsafe(ROW_BODY_SQL)} AS row,
|
|
744
814
|
1 - (e.embedding <=> ${literal}::vector) AS score,
|
|
745
815
|
e.salience ${halfLife !== void 0 && halfLife > 0 ? tx`* exp(-ln(2) * GREATEST(0, EXTRACT(EPOCH FROM (now() - e.salience_updated_at))) / ${halfLife * 3600})` : tx``} AS salience
|
|
746
816
|
${wantVector ? tx`, e.embedding::text AS embedding` : tx``}
|
|
747
817
|
FROM embeddings e
|
|
748
|
-
JOIN
|
|
818
|
+
JOIN ${rowsFrom(tx, scoped, true)} r
|
|
749
819
|
ON r.namespace = e.row_namespace
|
|
750
820
|
AND r.type = e.row_type
|
|
751
821
|
AND r.id = e.row_id
|
|
@@ -754,11 +824,11 @@ var PgEmbeddingsStore = class {
|
|
|
754
824
|
AND e.model = ${input.model}
|
|
755
825
|
AND r.deleted_at IS NULL
|
|
756
826
|
AND 1 - (e.embedding <=> ${literal}::vector) >= ${threshold}
|
|
757
|
-
${rowFilterSql(tx, input.rowFilter)}
|
|
827
|
+
${rowFilterSql(tx, ref, input.rowFilter)}
|
|
758
828
|
ORDER BY e.embedding <=> ${literal}::vector
|
|
759
829
|
LIMIT ${k}
|
|
760
|
-
|
|
761
|
-
})).map((r) => ({
|
|
830
|
+
`);
|
|
831
|
+
}), "row")).map((r) => ({
|
|
762
832
|
row: r.row,
|
|
763
833
|
score: r.score,
|
|
764
834
|
salience: r.salience,
|
|
@@ -795,21 +865,28 @@ var PgEmbeddingsStore = class {
|
|
|
795
865
|
async fetchEmbeddings(ids, model) {
|
|
796
866
|
if (ids.length === 0) return [];
|
|
797
867
|
const sql = this.#sql;
|
|
798
|
-
|
|
868
|
+
const byType = /* @__PURE__ */ new Map();
|
|
869
|
+
for (const id of ids) {
|
|
870
|
+
const key = `${id.namespace}/${id.type}`;
|
|
871
|
+
byType.set(key, [...byType.get(key) ?? [], id]);
|
|
872
|
+
}
|
|
873
|
+
const scoped = await scopedRefs(sql, [...byType.values()].map(([first]) => first));
|
|
874
|
+
const fetch = (part, from) => sql`
|
|
799
875
|
SELECT e.row_namespace, e.row_type, e.row_id, e.embedding::text AS embedding,
|
|
800
876
|
${sql.unsafe(ROW_BODY_SQL)} AS body, e.salience
|
|
801
|
-
FROM (VALUES ${sql(
|
|
877
|
+
FROM (VALUES ${sql(part.map((id) => [
|
|
802
878
|
id.namespace,
|
|
803
879
|
id.type,
|
|
804
880
|
id.id
|
|
805
881
|
]))}) AS t(ns, ty, id)
|
|
806
882
|
JOIN embeddings e
|
|
807
883
|
ON e.row_namespace = t.ns AND e.row_type = t.ty AND e.row_id = t.id
|
|
808
|
-
JOIN
|
|
884
|
+
JOIN ${from} r
|
|
809
885
|
ON r.namespace = e.row_namespace AND r.type = e.row_type AND r.id = e.row_id
|
|
810
886
|
WHERE e.model = ${model}
|
|
811
887
|
AND r.deleted_at IS NULL
|
|
812
|
-
|
|
888
|
+
`;
|
|
889
|
+
return (await Promise.all(scoped.filter((s) => s.ref !== void 0).map((s) => fetch(byType.get(`${s.namespace}/${s.type}`) ?? [], rowsFrom(sql, [s]))))).flat().map((r) => ({
|
|
813
890
|
namespace: r.row_namespace,
|
|
814
891
|
type: r.row_type,
|
|
815
892
|
id: r.row_id,
|
|
@@ -820,40 +897,39 @@ var PgEmbeddingsStore = class {
|
|
|
820
897
|
}
|
|
821
898
|
async selectCandidates(selector, model) {
|
|
822
899
|
const sql = this.#sql;
|
|
823
|
-
const
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
900
|
+
const arms = (await scopedRefs(sql, selector.scopes)).flatMap((s) => s.ref === void 0 ? [] : [{
|
|
901
|
+
...s,
|
|
902
|
+
ref: s.ref
|
|
903
|
+
}]);
|
|
904
|
+
if (arms.length === 0) return [];
|
|
905
|
+
let kept = sql``;
|
|
906
|
+
for (const [i, { namespace, type, ref }] of arms.entries()) {
|
|
907
|
+
let keep = sql`x.deleted_at IS NULL`;
|
|
908
|
+
for (const { field, prefix } of selector.excludeTextPrefixes) keep = sql`${keep} AND NOT COALESCE(
|
|
909
|
+
jsonb_typeof(${sql.unsafe(ref.json("x", field))}) IN ('string', 'number', 'boolean')
|
|
910
|
+
AND starts_with(${sql.unsafe(ref.text("x", field))}, ${prefix}::text), false)`;
|
|
911
|
+
kept = sql`${kept}${sql.unsafe(i === 0 ? "" : " UNION ALL ")}SELECT ${namespace}::text AS namespace, ${type}::text AS type, x.id FROM ${sql.unsafe(ref.from)} x WHERE ${keep}`;
|
|
912
|
+
}
|
|
827
913
|
return (await sql`
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
-- to scalar JSON values (jsonb_typeof) to match the in-memory side,
|
|
848
|
-
-- which coerces only string/number/boolean via String(value) — a
|
|
849
|
-
-- composite (object/array) has no agreed-on text rendering between
|
|
850
|
-
-- pg's jsonb ->> and JS's String(), so both sides now treat a
|
|
851
|
-
-- composite as never-excluded rather than risk disagreeing.
|
|
852
|
-
WHERE jsonb_typeof(${sql.unsafe(ROW_BODY_SQL)} -> ex.field) IN ('string', 'number', 'boolean')
|
|
853
|
-
AND starts_with(${sql.unsafe(ROW_BODY_SQL)} ->> ex.field, ex.prefix)
|
|
854
|
-
)
|
|
855
|
-
ORDER BY e.row_namespace COLLATE "C" ASC, e.row_type COLLATE "C" ASC, e.row_id COLLATE "C" ASC
|
|
856
|
-
LIMIT ${selector.limit}
|
|
914
|
+
WITH picked AS (
|
|
915
|
+
SELECT e.row_namespace, e.row_type, e.row_id
|
|
916
|
+
FROM embeddings e
|
|
917
|
+
JOIN (${kept}) k
|
|
918
|
+
ON k.namespace = e.row_namespace AND k.type = e.row_type AND k.id = e.row_id
|
|
919
|
+
WHERE e.model = ${model}
|
|
920
|
+
ORDER BY e.row_namespace COLLATE "C" ASC, e.row_type COLLATE "C" ASC, e.row_id COLLATE "C" ASC
|
|
921
|
+
LIMIT ${selector.limit}
|
|
922
|
+
)
|
|
923
|
+
SELECT p.row_namespace, p.row_type, p.row_id, e.embedding::text AS embedding, b.body
|
|
924
|
+
FROM picked p
|
|
925
|
+
JOIN embeddings e
|
|
926
|
+
ON e.row_namespace = p.row_namespace AND e.row_type = p.row_type AND e.row_id = p.row_id
|
|
927
|
+
CROSS JOIN LATERAL (
|
|
928
|
+
SELECT ${sql.unsafe(ROW_BODY_SQL)} AS body FROM ${rowsFrom(sql, arms)} r
|
|
929
|
+
WHERE r.namespace = p.row_namespace AND r.type = p.row_type AND r.id = p.row_id
|
|
930
|
+
OFFSET 0
|
|
931
|
+
) b
|
|
932
|
+
ORDER BY p.row_namespace COLLATE "C" ASC, p.row_type COLLATE "C" ASC, p.row_id COLLATE "C" ASC
|
|
857
933
|
`).map((r) => ({
|
|
858
934
|
namespace: r.row_namespace,
|
|
859
935
|
type: r.row_type,
|
|
@@ -862,17 +938,16 @@ var PgEmbeddingsStore = class {
|
|
|
862
938
|
row: r.body
|
|
863
939
|
}));
|
|
864
940
|
}
|
|
865
|
-
/**
|
|
866
|
-
* Deletes every embedding whose backing row is missing or soft-deleted.
|
|
867
|
-
* Needed because `embeddings` has no FK to `rows` and no
|
|
868
|
-
* dataset delete path touches it; tombstones are terminal, so soft-deleted
|
|
869
|
-
* rows are safe to collect immediately. Run as a sweep stage by the binary.
|
|
870
|
-
*
|
|
871
|
-
* @returns number of orphaned embeddings deleted.
|
|
872
|
-
*/
|
|
873
941
|
async staleRowIds(input) {
|
|
874
|
-
|
|
875
|
-
|
|
942
|
+
const sql = this.#sql;
|
|
943
|
+
const ref = await typeTableRef(sql, input.namespace, input.type);
|
|
944
|
+
if (ref === void 0) return [];
|
|
945
|
+
return (await sql`
|
|
946
|
+
SELECT r.id FROM ${rowsFrom(sql, [{
|
|
947
|
+
namespace: input.namespace,
|
|
948
|
+
type: input.type,
|
|
949
|
+
ref
|
|
950
|
+
}])} r
|
|
876
951
|
WHERE r.namespace = ${input.namespace}
|
|
877
952
|
AND r.type = ${input.type}
|
|
878
953
|
AND r.deleted_at IS NULL
|
|
@@ -888,22 +963,29 @@ var PgEmbeddingsStore = class {
|
|
|
888
963
|
LIMIT ${input.limit}
|
|
889
964
|
`).map((r) => r.id);
|
|
890
965
|
}
|
|
966
|
+
/**
|
|
967
|
+
* Deletes every embedding whose backing row is missing or soft-deleted.
|
|
968
|
+
* Needed because `embeddings` has no FK to the type tables and no
|
|
969
|
+
* dataset delete path touches it; tombstones are terminal, so soft-deleted
|
|
970
|
+
* rows are safe to collect immediately. Run as a sweep stage by the binary.
|
|
971
|
+
*
|
|
972
|
+
* @returns number of orphaned embeddings deleted.
|
|
973
|
+
*/
|
|
891
974
|
async purgeOrphans() {
|
|
892
|
-
return (
|
|
893
|
-
DELETE FROM embeddings e
|
|
894
|
-
WHERE NOT EXISTS (
|
|
895
|
-
SELECT 1 FROM rows r
|
|
896
|
-
WHERE r.namespace = e.row_namespace
|
|
897
|
-
AND r.type = e.row_type
|
|
898
|
-
AND r.id = e.row_id
|
|
899
|
-
AND r.deleted_at IS NULL
|
|
900
|
-
)
|
|
901
|
-
`).count;
|
|
975
|
+
return purgeTypeTableOrphans(this.#sql, "embeddings");
|
|
902
976
|
}
|
|
903
977
|
};
|
|
904
978
|
|
|
905
979
|
//#endregion
|
|
906
980
|
//#region ../../plugins/substrate-recall/src/pg-lexical-store.ts
|
|
981
|
+
/**
|
|
982
|
+
* Postgres-backed `LexicalStore` — `lexical` + `ts_rank` over its GIN
|
|
983
|
+
* index. Counterpart to `PgEmbeddingsStore`.
|
|
984
|
+
*
|
|
985
|
+
* Search joins the type's rows the same way the KNN path does, with the same
|
|
986
|
+
* `deleted_at IS NULL` predicate, so a deleted row cannot surface through this
|
|
987
|
+
* arm after it has stopped surfacing through the dense one.
|
|
988
|
+
*/
|
|
907
989
|
/** Hard ceiling on `k`, mirroring the embeddings store's cap. */
|
|
908
990
|
const MAX_K = 100;
|
|
909
991
|
var PgLexicalStore = class {
|
|
@@ -933,27 +1015,24 @@ var PgLexicalStore = class {
|
|
|
933
1015
|
/**
|
|
934
1016
|
* Deletes every lexical row whose backing row is missing or soft-deleted,
|
|
935
1017
|
* as `PgEmbeddingsStore.purgeOrphans` does for vectors: `lexical` has no FK
|
|
936
|
-
* to
|
|
1018
|
+
* to the type tables, so a drop, purge or tombstone would otherwise leave the text.
|
|
937
1019
|
*
|
|
938
1020
|
* @returns number of orphaned lexical rows deleted.
|
|
939
1021
|
*/
|
|
940
1022
|
async purgeOrphans() {
|
|
941
|
-
return (
|
|
942
|
-
DELETE FROM lexical l
|
|
943
|
-
WHERE NOT EXISTS (
|
|
944
|
-
SELECT 1 FROM rows r
|
|
945
|
-
WHERE r.namespace = l.row_namespace
|
|
946
|
-
AND r.type = l.row_type
|
|
947
|
-
AND r.id = l.row_id
|
|
948
|
-
AND r.deleted_at IS NULL
|
|
949
|
-
)
|
|
950
|
-
`).count;
|
|
1023
|
+
return purgeTypeTableOrphans(this.#sql, "lexical");
|
|
951
1024
|
}
|
|
952
1025
|
async countIndexed(namespace, type) {
|
|
1026
|
+
const ref = await typeTableRef(this.#sql, namespace, type);
|
|
1027
|
+
if (ref === void 0) return 0;
|
|
953
1028
|
const rows = await this.#sql`
|
|
954
1029
|
SELECT COUNT(*) AS n
|
|
955
1030
|
FROM lexical l
|
|
956
|
-
JOIN
|
|
1031
|
+
JOIN ${rowsFrom(this.#sql, [{
|
|
1032
|
+
namespace,
|
|
1033
|
+
type,
|
|
1034
|
+
ref
|
|
1035
|
+
}])} r
|
|
957
1036
|
ON r.namespace = l.row_namespace
|
|
958
1037
|
AND r.type = l.row_type
|
|
959
1038
|
AND r.id = l.row_id
|
|
@@ -979,6 +1058,7 @@ var PgLexicalStore = class {
|
|
|
979
1058
|
if (strict.length > 0) return strict;
|
|
980
1059
|
return this.#run(input, k, true);
|
|
981
1060
|
}
|
|
1061
|
+
/** One pass, under the type table's read guard, so a promotion meanwhile cannot drop a field or a hit. */
|
|
982
1062
|
async #run(input, k, degrade) {
|
|
983
1063
|
const query = degrade ? this.#sql`(
|
|
984
1064
|
SELECT CASE
|
|
@@ -987,7 +1067,7 @@ var PgLexicalStore = class {
|
|
|
987
1067
|
END AS query
|
|
988
1068
|
FROM (SELECT websearch_to_tsquery(${this.#tsConfig}, ${input.query})::text AS t) s
|
|
989
1069
|
)` : this.#sql`websearch_to_tsquery(${this.#tsConfig}, ${input.query})`;
|
|
990
|
-
return (await this.#sql`
|
|
1070
|
+
return (await guardedRefRead(this.#sql, input.namespace, input.type, (ref, guard) => guard(this.#sql, this.#sql`
|
|
991
1071
|
SELECT
|
|
992
1072
|
l.row_namespace,
|
|
993
1073
|
l.row_type,
|
|
@@ -996,7 +1076,11 @@ var PgLexicalStore = class {
|
|
|
996
1076
|
ts_rank(l.tsv, q.query) AS score
|
|
997
1077
|
FROM lexical l
|
|
998
1078
|
CROSS JOIN ${query} AS q(query)
|
|
999
|
-
JOIN
|
|
1079
|
+
JOIN ${rowsFrom(this.#sql, [{
|
|
1080
|
+
namespace: input.namespace,
|
|
1081
|
+
type: input.type,
|
|
1082
|
+
ref
|
|
1083
|
+
}], true)} r
|
|
1000
1084
|
ON r.namespace = l.row_namespace
|
|
1001
1085
|
AND r.type = l.row_type
|
|
1002
1086
|
AND r.id = l.row_id
|
|
@@ -1005,10 +1089,10 @@ var PgLexicalStore = class {
|
|
|
1005
1089
|
AND r.deleted_at IS NULL
|
|
1006
1090
|
AND q.query IS NOT NULL
|
|
1007
1091
|
AND l.tsv @@ q.query
|
|
1008
|
-
${rowFilterSql(this.#sql, input.rowFilter)}
|
|
1092
|
+
${rowFilterSql(this.#sql, ref, input.rowFilter)}
|
|
1009
1093
|
ORDER BY score DESC, l.row_id ASC
|
|
1010
1094
|
LIMIT ${k}
|
|
1011
|
-
`).map((r) => ({
|
|
1095
|
+
`), "row_id")).map((r) => ({
|
|
1012
1096
|
rowNamespace: r.row_namespace,
|
|
1013
1097
|
rowType: r.row_type,
|
|
1014
1098
|
rowId: r.row_id,
|