@aigne/afs-index 1.12.0-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +26 -0
- package/dist/_virtual/_@oxc-project_runtime@0.108.0/helpers/decorate.cjs +11 -0
- package/dist/_virtual/_@oxc-project_runtime@0.108.0/helpers/decorate.mjs +10 -0
- package/dist/backend/d1-index-backend.cjs +656 -0
- package/dist/backend/d1-index-backend.d.cts +149 -0
- package/dist/backend/d1-index-backend.d.cts.map +1 -0
- package/dist/backend/d1-index-backend.d.mts +149 -0
- package/dist/backend/d1-index-backend.d.mts.map +1 -0
- package/dist/backend/d1-index-backend.mjs +657 -0
- package/dist/backend/d1-index-backend.mjs.map +1 -0
- package/dist/backend/d1-vector-backend.cjs +83 -0
- package/dist/backend/d1-vector-backend.d.cts +21 -0
- package/dist/backend/d1-vector-backend.d.cts.map +1 -0
- package/dist/backend/d1-vector-backend.d.mts +21 -0
- package/dist/backend/d1-vector-backend.d.mts.map +1 -0
- package/dist/backend/d1-vector-backend.mjs +84 -0
- package/dist/backend/d1-vector-backend.mjs.map +1 -0
- package/dist/backend/memory-backend.cjs +304 -0
- package/dist/backend/memory-backend.d.cts +66 -0
- package/dist/backend/memory-backend.d.cts.map +1 -0
- package/dist/backend/memory-backend.d.mts +66 -0
- package/dist/backend/memory-backend.d.mts.map +1 -0
- package/dist/backend/memory-backend.mjs +305 -0
- package/dist/backend/memory-backend.mjs.map +1 -0
- package/dist/backend/prefix-range.cjs +25 -0
- package/dist/backend/prefix-range.mjs +25 -0
- package/dist/backend/prefix-range.mjs.map +1 -0
- package/dist/backend/schema.cjs +187 -0
- package/dist/backend/schema.mjs +188 -0
- package/dist/backend/schema.mjs.map +1 -0
- package/dist/backend/sqlite-backend.cjs +889 -0
- package/dist/backend/sqlite-backend.d.cts +96 -0
- package/dist/backend/sqlite-backend.d.cts.map +1 -0
- package/dist/backend/sqlite-backend.d.mts +96 -0
- package/dist/backend/sqlite-backend.d.mts.map +1 -0
- package/dist/backend/sqlite-backend.mjs +890 -0
- package/dist/backend/sqlite-backend.mjs.map +1 -0
- package/dist/backend/types.d.cts +209 -0
- package/dist/backend/types.d.cts.map +1 -0
- package/dist/backend/types.d.mts +209 -0
- package/dist/backend/types.d.mts.map +1 -0
- package/dist/backend/upstash-vector-backend.cjs +98 -0
- package/dist/backend/upstash-vector-backend.d.cts +21 -0
- package/dist/backend/upstash-vector-backend.d.cts.map +1 -0
- package/dist/backend/upstash-vector-backend.d.mts +21 -0
- package/dist/backend/upstash-vector-backend.d.mts.map +1 -0
- package/dist/backend/upstash-vector-backend.mjs +98 -0
- package/dist/backend/upstash-vector-backend.mjs.map +1 -0
- package/dist/backend/vector-backend.d.cts +29 -0
- package/dist/backend/vector-backend.d.cts.map +1 -0
- package/dist/backend/vector-backend.d.mts +29 -0
- package/dist/backend/vector-backend.d.mts.map +1 -0
- package/dist/dirty-queue.d.cts +41 -0
- package/dist/dirty-queue.d.cts.map +1 -0
- package/dist/dirty-queue.d.mts +41 -0
- package/dist/dirty-queue.d.mts.map +1 -0
- package/dist/embed.cjs +45 -0
- package/dist/embed.d.cts +20 -0
- package/dist/embed.d.cts.map +1 -0
- package/dist/embed.d.mts +20 -0
- package/dist/embed.d.mts.map +1 -0
- package/dist/embed.mjs +46 -0
- package/dist/embed.mjs.map +1 -0
- package/dist/extraction/defaults.cjs +242 -0
- package/dist/extraction/defaults.d.cts +14 -0
- package/dist/extraction/defaults.d.cts.map +1 -0
- package/dist/extraction/defaults.d.mts +14 -0
- package/dist/extraction/defaults.d.mts.map +1 -0
- package/dist/extraction/defaults.mjs +242 -0
- package/dist/extraction/defaults.mjs.map +1 -0
- package/dist/extraction/field-map.cjs +162 -0
- package/dist/extraction/field-map.d.cts +70 -0
- package/dist/extraction/field-map.d.cts.map +1 -0
- package/dist/extraction/field-map.d.mts +70 -0
- package/dist/extraction/field-map.d.mts.map +1 -0
- package/dist/extraction/field-map.mjs +157 -0
- package/dist/extraction/field-map.mjs.map +1 -0
- package/dist/extraction/manager.cjs +74 -0
- package/dist/extraction/manager.d.cts +47 -0
- package/dist/extraction/manager.d.cts.map +1 -0
- package/dist/extraction/manager.d.mts +47 -0
- package/dist/extraction/manager.d.mts.map +1 -0
- package/dist/extraction/manager.mjs +74 -0
- package/dist/extraction/manager.mjs.map +1 -0
- package/dist/extraction/path-anchors.cjs +86 -0
- package/dist/extraction/path-anchors.d.cts +72 -0
- package/dist/extraction/path-anchors.d.cts.map +1 -0
- package/dist/extraction/path-anchors.d.mts +72 -0
- package/dist/extraction/path-anchors.d.mts.map +1 -0
- package/dist/extraction/path-anchors.mjs +86 -0
- package/dist/extraction/path-anchors.mjs.map +1 -0
- package/dist/follower-scheduler.cjs +92 -0
- package/dist/follower-scheduler.d.cts +58 -0
- package/dist/follower-scheduler.d.cts.map +1 -0
- package/dist/follower-scheduler.d.mts +58 -0
- package/dist/follower-scheduler.d.mts.map +1 -0
- package/dist/follower-scheduler.mjs +92 -0
- package/dist/follower-scheduler.mjs.map +1 -0
- package/dist/follower.cjs +577 -0
- package/dist/follower.d.cts +241 -0
- package/dist/follower.d.cts.map +1 -0
- package/dist/follower.d.mts +241 -0
- package/dist/follower.d.mts.map +1 -0
- package/dist/follower.mjs +571 -0
- package/dist/follower.mjs.map +1 -0
- package/dist/index-provider.cjs +1166 -0
- package/dist/index-provider.d.cts +175 -0
- package/dist/index-provider.d.cts.map +1 -0
- package/dist/index-provider.d.mts +175 -0
- package/dist/index-provider.d.mts.map +1 -0
- package/dist/index-provider.mjs +1167 -0
- package/dist/index-provider.mjs.map +1 -0
- package/dist/index.cjs +53 -0
- package/dist/index.d.cts +22 -0
- package/dist/index.d.mts +22 -0
- package/dist/index.mjs +20 -0
- package/dist/keys/exact.cjs +20 -0
- package/dist/keys/exact.mjs +20 -0
- package/dist/keys/exact.mjs.map +1 -0
- package/dist/keys/location.cjs +13 -0
- package/dist/keys/location.mjs +13 -0
- package/dist/keys/location.mjs.map +1 -0
- package/dist/keys/normalized.cjs +62 -0
- package/dist/keys/normalized.mjs +63 -0
- package/dist/keys/normalized.mjs.map +1 -0
- package/dist/keys/person.cjs +15 -0
- package/dist/keys/person.mjs +15 -0
- package/dist/keys/person.mjs.map +1 -0
- package/dist/keys/registry.cjs +45 -0
- package/dist/keys/registry.d.cts +15 -0
- package/dist/keys/registry.d.cts.map +1 -0
- package/dist/keys/registry.d.mts +15 -0
- package/dist/keys/registry.d.mts.map +1 -0
- package/dist/keys/registry.mjs +46 -0
- package/dist/keys/registry.mjs.map +1 -0
- package/dist/keys/time.cjs +128 -0
- package/dist/keys/time.mjs +128 -0
- package/dist/keys/time.mjs.map +1 -0
- package/dist/keys/topic.cjs +23 -0
- package/dist/keys/topic.mjs +23 -0
- package/dist/keys/topic.mjs.map +1 -0
- package/dist/manifest-index.cjs +29 -0
- package/dist/manifest-index.d.cts +24 -0
- package/dist/manifest-index.d.cts.map +1 -0
- package/dist/manifest-index.d.mts +24 -0
- package/dist/manifest-index.d.mts.map +1 -0
- package/dist/manifest-index.mjs +28 -0
- package/dist/manifest-index.mjs.map +1 -0
- package/dist/offline-rebuild.cjs +48 -0
- package/dist/offline-rebuild.d.cts +57 -0
- package/dist/offline-rebuild.d.cts.map +1 -0
- package/dist/offline-rebuild.d.mts +57 -0
- package/dist/offline-rebuild.d.mts.map +1 -0
- package/dist/offline-rebuild.mjs +49 -0
- package/dist/offline-rebuild.mjs.map +1 -0
- package/dist/query/engine.cjs +433 -0
- package/dist/query/engine.d.cts +34 -0
- package/dist/query/engine.d.cts.map +1 -0
- package/dist/query/engine.d.mts +34 -0
- package/dist/query/engine.d.mts.map +1 -0
- package/dist/query/engine.mjs +433 -0
- package/dist/query/engine.mjs.map +1 -0
- package/dist/query/limits.cjs +12 -0
- package/dist/query/limits.mjs +12 -0
- package/dist/query/limits.mjs.map +1 -0
- package/dist/text/cjk.cjs +30 -0
- package/dist/text/cjk.d.cts +24 -0
- package/dist/text/cjk.d.cts.map +1 -0
- package/dist/text/cjk.d.mts +24 -0
- package/dist/text/cjk.d.mts.map +1 -0
- package/dist/text/cjk.mjs +30 -0
- package/dist/text/cjk.mjs.map +1 -0
- package/dist/text/fts.cjs +244 -0
- package/dist/text/fts.mjs +245 -0
- package/dist/text/fts.mjs.map +1 -0
- package/dist/text/host.cjs +39 -0
- package/dist/text/host.mjs +38 -0
- package/dist/text/host.mjs.map +1 -0
- package/dist/types.cjs +44 -0
- package/dist/types.d.cts +393 -0
- package/dist/types.d.cts.map +1 -0
- package/dist/types.d.mts +393 -0
- package/dist/types.d.mts.map +1 -0
- package/dist/types.mjs +42 -0
- package/dist/types.mjs.map +1 -0
- package/package.json +59 -0
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { IndexBackend } from "./backend/types.cjs";
|
|
2
|
+
import { TimeKeyGranularity } from "./types.cjs";
|
|
3
|
+
import { AFSRoot, BlockletIndexDomain } from "@aigne/afs";
|
|
4
|
+
|
|
5
|
+
//#region src/offline-rebuild.d.ts
|
|
6
|
+
interface OfflineRebuildResult {
|
|
7
|
+
/** Relational/FTS/anchor rows cleared before the rebuild (vectors preserved). */
|
|
8
|
+
cleared: number;
|
|
9
|
+
/** Live paths (re)indexed. */
|
|
10
|
+
indexed: number;
|
|
11
|
+
/** Live paths that failed to index (kept for the caller to log). */
|
|
12
|
+
failed: number;
|
|
13
|
+
/** Live paths no declared domain claims (skipped). */
|
|
14
|
+
skipped: number;
|
|
15
|
+
}
|
|
16
|
+
interface OfflineRebuildOptions {
|
|
17
|
+
/** Scope-stamped index backend (writes relational/FTS/anchor for one scope). */
|
|
18
|
+
backend: IndexBackend;
|
|
19
|
+
/** Content-reading AFS for the scope's fragment (`read`/`stat` of live files). */
|
|
20
|
+
spaceAfs: AFSRoot;
|
|
21
|
+
/** The blocklet's declared index domains. */
|
|
22
|
+
domains: BlockletIndexDomain[];
|
|
23
|
+
/**
|
|
24
|
+
* Space-relative live entry paths to rebuild (e.g. `items/x.json`) — enumerated
|
|
25
|
+
* by the caller from live `ds_entries` under `blocklets/{instanceDid}/user/`.
|
|
26
|
+
*/
|
|
27
|
+
livePaths: string[];
|
|
28
|
+
/**
|
|
29
|
+
* The blocklet's `index.timeKeyGranularity` (blocklet.yaml top-level, NOT part
|
|
30
|
+
* of `domains[]`) — caps which time buckets a `time` anchor expands into
|
|
31
|
+
* (issue #1215; e.g. `[year, month, day]` instead of the 8-bucket default).
|
|
32
|
+
*
|
|
33
|
+
* CRITICAL for a config-change rebuild: the `time` key expansion happens in the
|
|
34
|
+
* BACKEND's `keyRegistry` (`generateKeys`), which a plain `new D1IndexBackend`
|
|
35
|
+
* default-constructs to the FULL 8-bucket generator. The live/cron path applies
|
|
36
|
+
* the policy by having `AFSIndex` register a granularity-limited `time`
|
|
37
|
+
* generator and `setKeyRegistry` it onto the backend (index-provider.ts) — the
|
|
38
|
+
* offline seam must do the SAME, or a rebuild silently re-writes the old
|
|
39
|
+
* high-cardinality time keys. When set, this wires that generator before the
|
|
40
|
+
* clear+rebuild. Omit to keep the backend's existing registry unchanged.
|
|
41
|
+
*/
|
|
42
|
+
timeKeyGranularity?: TimeKeyGranularity[];
|
|
43
|
+
/**
|
|
44
|
+
* Optional progress reporter for long (esp. REMOTE) rebuilds. `phase:"clear"`
|
|
45
|
+
* fires once with the count about to be purged; `phase:"rebuild"` fires
|
|
46
|
+
* periodically with `done`/`total` reindexed paths.
|
|
47
|
+
*/
|
|
48
|
+
onProgress?: (info: {
|
|
49
|
+
phase: "clear" | "rebuild";
|
|
50
|
+
done: number;
|
|
51
|
+
total: number;
|
|
52
|
+
}) => void;
|
|
53
|
+
}
|
|
54
|
+
declare function offlineRebuildScope(opts: OfflineRebuildOptions): Promise<OfflineRebuildResult>;
|
|
55
|
+
//#endregion
|
|
56
|
+
export { OfflineRebuildOptions, OfflineRebuildResult, offlineRebuildScope };
|
|
57
|
+
//# sourceMappingURL=offline-rebuild.d.cts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"offline-rebuild.d.cts","names":[],"sources":["../src/offline-rebuild.ts"],"mappings":";;;;;UAgCiB,oBAAA;EAiBf;EAfA,OAAA;EAoBA;EAlBA,OAAA;EAiCqB;EA/BrB,MAAA;EAqCsB;EAnCtB,OAAA;AAAA;AAAA,UAGe,qBAAA;EAgCgE;EA9B/E,OAAA,EAAS,YAAA;EAiCW;EA/BpB,QAAA,EAAU,OAAA;;EAEV,OAAA,EAAS,mBAAA;EA+BA;;;;EA1BT,SAAA;EAyBA;;;;;;;;;;;;;;EAVA,kBAAA,GAAqB,kBAAA;;;;;;EAMrB,UAAA,IAAc,IAAA;IAAQ,KAAA;IAA4B,IAAA;IAAc,KAAA;EAAA;AAAA;AAAA,iBAG5C,mBAAA,CACpB,IAAA,EAAM,qBAAA,GACL,OAAA,CAAQ,oBAAA"}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { IndexBackend } from "./backend/types.mjs";
|
|
2
|
+
import { TimeKeyGranularity } from "./types.mjs";
|
|
3
|
+
import { AFSRoot, BlockletIndexDomain } from "@aigne/afs";
|
|
4
|
+
|
|
5
|
+
//#region src/offline-rebuild.d.ts
|
|
6
|
+
interface OfflineRebuildResult {
|
|
7
|
+
/** Relational/FTS/anchor rows cleared before the rebuild (vectors preserved). */
|
|
8
|
+
cleared: number;
|
|
9
|
+
/** Live paths (re)indexed. */
|
|
10
|
+
indexed: number;
|
|
11
|
+
/** Live paths that failed to index (kept for the caller to log). */
|
|
12
|
+
failed: number;
|
|
13
|
+
/** Live paths no declared domain claims (skipped). */
|
|
14
|
+
skipped: number;
|
|
15
|
+
}
|
|
16
|
+
interface OfflineRebuildOptions {
|
|
17
|
+
/** Scope-stamped index backend (writes relational/FTS/anchor for one scope). */
|
|
18
|
+
backend: IndexBackend;
|
|
19
|
+
/** Content-reading AFS for the scope's fragment (`read`/`stat` of live files). */
|
|
20
|
+
spaceAfs: AFSRoot;
|
|
21
|
+
/** The blocklet's declared index domains. */
|
|
22
|
+
domains: BlockletIndexDomain[];
|
|
23
|
+
/**
|
|
24
|
+
* Space-relative live entry paths to rebuild (e.g. `items/x.json`) — enumerated
|
|
25
|
+
* by the caller from live `ds_entries` under `blocklets/{instanceDid}/user/`.
|
|
26
|
+
*/
|
|
27
|
+
livePaths: string[];
|
|
28
|
+
/**
|
|
29
|
+
* The blocklet's `index.timeKeyGranularity` (blocklet.yaml top-level, NOT part
|
|
30
|
+
* of `domains[]`) — caps which time buckets a `time` anchor expands into
|
|
31
|
+
* (issue #1215; e.g. `[year, month, day]` instead of the 8-bucket default).
|
|
32
|
+
*
|
|
33
|
+
* CRITICAL for a config-change rebuild: the `time` key expansion happens in the
|
|
34
|
+
* BACKEND's `keyRegistry` (`generateKeys`), which a plain `new D1IndexBackend`
|
|
35
|
+
* default-constructs to the FULL 8-bucket generator. The live/cron path applies
|
|
36
|
+
* the policy by having `AFSIndex` register a granularity-limited `time`
|
|
37
|
+
* generator and `setKeyRegistry` it onto the backend (index-provider.ts) — the
|
|
38
|
+
* offline seam must do the SAME, or a rebuild silently re-writes the old
|
|
39
|
+
* high-cardinality time keys. When set, this wires that generator before the
|
|
40
|
+
* clear+rebuild. Omit to keep the backend's existing registry unchanged.
|
|
41
|
+
*/
|
|
42
|
+
timeKeyGranularity?: TimeKeyGranularity[];
|
|
43
|
+
/**
|
|
44
|
+
* Optional progress reporter for long (esp. REMOTE) rebuilds. `phase:"clear"`
|
|
45
|
+
* fires once with the count about to be purged; `phase:"rebuild"` fires
|
|
46
|
+
* periodically with `done`/`total` reindexed paths.
|
|
47
|
+
*/
|
|
48
|
+
onProgress?: (info: {
|
|
49
|
+
phase: "clear" | "rebuild";
|
|
50
|
+
done: number;
|
|
51
|
+
total: number;
|
|
52
|
+
}) => void;
|
|
53
|
+
}
|
|
54
|
+
declare function offlineRebuildScope(opts: OfflineRebuildOptions): Promise<OfflineRebuildResult>;
|
|
55
|
+
//#endregion
|
|
56
|
+
export { OfflineRebuildOptions, OfflineRebuildResult, offlineRebuildScope };
|
|
57
|
+
//# sourceMappingURL=offline-rebuild.d.mts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"offline-rebuild.d.mts","names":[],"sources":["../src/offline-rebuild.ts"],"mappings":";;;;;UAgCiB,oBAAA;EAiBf;EAfA,OAAA;EAoBA;EAlBA,OAAA;EAiCqB;EA/BrB,MAAA;EAqCsB;EAnCtB,OAAA;AAAA;AAAA,UAGe,qBAAA;EAgCgE;EA9B/E,OAAA,EAAS,YAAA;EAiCW;EA/BpB,QAAA,EAAU,OAAA;;EAEV,OAAA,EAAS,mBAAA;EA+BA;;;;EA1BT,SAAA;EAyBA;;;;;;;;;;;;;;EAVA,kBAAA,GAAqB,kBAAA;;;;;;EAMrB,UAAA,IAAc,IAAA;IAAQ,KAAA;IAA4B,IAAA;IAAc,KAAA;EAAA;AAAA;AAAA,iBAG5C,mBAAA,CACpB,IAAA,EAAM,qBAAA,GACL,OAAA,CAAQ,oBAAA"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { generateTimeKeys } from "./keys/time.mjs";
|
|
2
|
+
import { KeyGeneratorRegistry } from "./keys/registry.mjs";
|
|
3
|
+
import { IndexFollower } from "./follower.mjs";
|
|
4
|
+
|
|
5
|
+
//#region src/offline-rebuild.ts
|
|
6
|
+
async function offlineRebuildScope(opts) {
|
|
7
|
+
const { backend, spaceAfs, domains, livePaths, timeKeyGranularity } = opts;
|
|
8
|
+
if (timeKeyGranularity?.length && backend.setKeyRegistry) {
|
|
9
|
+
const registry = new KeyGeneratorRegistry();
|
|
10
|
+
const include = new Set(timeKeyGranularity);
|
|
11
|
+
registry.register("time", (anchor) => generateTimeKeys(anchor, include));
|
|
12
|
+
backend.setKeyRegistry(registry);
|
|
13
|
+
}
|
|
14
|
+
let cleared = 0;
|
|
15
|
+
const boundDomains = domains.map((d) => d.name);
|
|
16
|
+
const existing = backend.listEntries ? await backend.listEntries({ boundDomains }) : [];
|
|
17
|
+
if (existing.length > 0 && backend.purgeEntries) {
|
|
18
|
+
opts.onProgress?.({
|
|
19
|
+
phase: "clear",
|
|
20
|
+
done: 0,
|
|
21
|
+
total: existing.length
|
|
22
|
+
});
|
|
23
|
+
cleared = await backend.purgeEntries(existing.map((e) => e.entryPath), { skipVector: true });
|
|
24
|
+
opts.onProgress?.({
|
|
25
|
+
phase: "clear",
|
|
26
|
+
done: cleared,
|
|
27
|
+
total: existing.length
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
const { indexed, failed, skipped } = await new IndexFollower({
|
|
31
|
+
spaceAfs,
|
|
32
|
+
backend,
|
|
33
|
+
domains
|
|
34
|
+
}).reindexPaths(livePaths, (done, total) => opts.onProgress?.({
|
|
35
|
+
phase: "rebuild",
|
|
36
|
+
done,
|
|
37
|
+
total
|
|
38
|
+
}));
|
|
39
|
+
return {
|
|
40
|
+
cleared,
|
|
41
|
+
indexed,
|
|
42
|
+
failed,
|
|
43
|
+
skipped
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
//#endregion
|
|
48
|
+
export { offlineRebuildScope };
|
|
49
|
+
//# sourceMappingURL=offline-rebuild.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"offline-rebuild.mjs","names":[],"sources":["../src/offline-rebuild.ts"],"sourcesContent":["/**\n * Offline reset-and-rebuild for ONE scope (index-follower-dirty-path §10.1).\n *\n * The relational / FTS / anchor layers are pure derived views of `ds_entries`\n * and need no embedding, so they can be cleared and rebuilt from CURRENT live\n * state cheaply. This is BOTH the first-deploy backfill and the ONLY recovery\n * path (the big-bang cutover keeps no changelog-replay fallback — design §10):\n *\n * 1. CLEAR the relational/FTS/anchor rows for the scope with `skipVector:true`\n * — the D1-native `idx_embedding_vector` AND any injected Upstash vectors\n * are left UNTOUCHED (constraint B: never re-embed).\n * 2. REBUILD from the live entries via the follower's `reindexPaths` with NO\n * embedder wired — byte-identical relational/FTS/anchor output, zero vector\n * writes.\n *\n * Gap-proof: the rebuild reads LIVE state, never the changelog, so it does NOT\n * depend on any cursor water line `L`/`M` (design §10 cutover gap) — a live\n * entry that was never indexed is rebuilt regardless. Idempotent + re-runnable:\n * clearing then rebuilding from the same live set converges to the same rows.\n *\n * This helper is the single tested seam both the CF and Node offline ops scripts\n * call; the scripts only supply a scope-stamped `backend`, a content-reading\n * `spaceAfs`, the `domains`, and the live entry paths to rebuild.\n */\n\nimport type { AFSRoot, BlockletIndexDomain } from \"@aigne/afs\";\nimport type { IndexBackend } from \"./backend/types.js\";\nimport { IndexFollower } from \"./follower.js\";\nimport { KeyGeneratorRegistry } from \"./keys/registry.js\";\nimport { generateTimeKeys } from \"./keys/time.js\";\nimport type { TimeKeyGranularity } from \"./types.js\";\n\nexport interface OfflineRebuildResult {\n /** Relational/FTS/anchor rows cleared before the rebuild (vectors preserved). */\n cleared: number;\n /** Live paths (re)indexed. */\n indexed: number;\n /** Live paths that failed to index (kept for the caller to log). */\n failed: number;\n /** Live paths no declared domain claims (skipped). */\n skipped: number;\n}\n\nexport interface OfflineRebuildOptions {\n /** Scope-stamped index backend (writes relational/FTS/anchor for one scope). */\n backend: IndexBackend;\n /** Content-reading AFS for the scope's fragment (`read`/`stat` of live files). */\n spaceAfs: AFSRoot;\n /** The blocklet's declared index domains. */\n domains: BlockletIndexDomain[];\n /**\n * Space-relative live entry paths to rebuild (e.g. `items/x.json`) — enumerated\n * by the caller from live `ds_entries` under `blocklets/{instanceDid}/user/`.\n */\n livePaths: string[];\n /**\n * The blocklet's `index.timeKeyGranularity` (blocklet.yaml top-level, NOT part\n * of `domains[]`) — caps which time buckets a `time` anchor expands into\n * (issue #1215; e.g. `[year, month, day]` instead of the 8-bucket default).\n *\n * CRITICAL for a config-change rebuild: the `time` key expansion happens in the\n * BACKEND's `keyRegistry` (`generateKeys`), which a plain `new D1IndexBackend`\n * default-constructs to the FULL 8-bucket generator. The live/cron path applies\n * the policy by having `AFSIndex` register a granularity-limited `time`\n * generator and `setKeyRegistry` it onto the backend (index-provider.ts) — the\n * offline seam must do the SAME, or a rebuild silently re-writes the old\n * high-cardinality time keys. When set, this wires that generator before the\n * clear+rebuild. Omit to keep the backend's existing registry unchanged.\n */\n timeKeyGranularity?: TimeKeyGranularity[];\n /**\n * Optional progress reporter for long (esp. REMOTE) rebuilds. `phase:\"clear\"`\n * fires once with the count about to be purged; `phase:\"rebuild\"` fires\n * periodically with `done`/`total` reindexed paths.\n */\n onProgress?: (info: { phase: \"clear\" | \"rebuild\"; done: number; total: number }) => void;\n}\n\nexport async function offlineRebuildScope(\n opts: OfflineRebuildOptions,\n): Promise<OfflineRebuildResult> {\n const { backend, spaceAfs, domains, livePaths, timeKeyGranularity } = opts;\n\n // 0. Apply the domain's time-key granularity policy (issue #1215) BEFORE the\n // rebuild writes any anchors. The `time` anchor → `idx_anchor_key` expansion\n // lives in the backend's keyRegistry; a plain backend defaults to the full\n // 8-bucket generator, so without this a rebuild re-writes the OLD\n // high-cardinality keys. A fresh registry keeps the built-in\n // exact/location/person/topic defaults and overrides ONLY `time` with the\n // granularity-limited generator.\n //\n // NOTE: this offline path does NOT register the declared facet types\n // (tag/site/status/…) with `createNormalizedValueGenerator` the way the\n // live follower does (index-provider.ts `registerDomains`). That is SAFE\n // only because `generateExactKeys` — the registry's fallback for those\n // unregistered types — now emits the SAME canonical bare, lowercased\n // `<type>:<value>` key the normalized generator would (see keys/exact.ts).\n // Before that fix the fallback emitted `tag:{\"value\":\"…\"}` JSON, so an\n // offline rebuild silently rewrote every facet key into a form the query\n // side never matched — the whole reason facet search broke on a rebuilt\n // index. Do NOT \"restore\" a JSON exact fallback without re-registering the\n // facet types here.\n if (timeKeyGranularity?.length && backend.setKeyRegistry) {\n const registry = new KeyGeneratorRegistry();\n const include = new Set(timeKeyGranularity);\n registry.register(\"time\", (anchor) => generateTimeKeys(anchor, include));\n backend.setKeyRegistry(registry);\n }\n\n // 1. Clear relational/FTS/anchor for THIS blocklet's domains only, PRESERVING\n // all vectors. CRITICAL: a scope is SHARED across a user's blocklets (CF:\n // scope = HMAC(userDid); Node: the shared user-index.db), all stamped with\n // the same `scope` but different `domain` values. Clearing by scope alone\n // (`listEntries({})`) would WIPE sibling blocklets' index rows — a\n // cross-blocklet data loss (tasks.md §Security/§Data-leak). Bound the clear\n // to this blocklet's own domains, exactly as the query/reindex paths do\n // (AFSIndexOptions.boundDomains). The rebuild below is already domain-bound\n // (reindexPaths only indexes paths matching `domains`), so the two stay\n // symmetric and confined to this blocklet.\n let cleared = 0;\n const boundDomains = domains.map((d) => d.name);\n const existing = backend.listEntries ? await backend.listEntries({ boundDomains }) : [];\n if (existing.length > 0 && backend.purgeEntries) {\n opts.onProgress?.({ phase: \"clear\", done: 0, total: existing.length });\n cleared = await backend.purgeEntries(\n existing.map((e) => e.entryPath),\n { skipVector: true },\n );\n opts.onProgress?.({ phase: \"clear\", done: cleared, total: existing.length });\n }\n\n // 2. Rebuild from live entries with NO embedder (vectors untouched) and NO\n // dirty queue (reindexPaths is changelog-independent — it re-reads live\n // content per path and writes the three derived layers).\n const follower = new IndexFollower({ spaceAfs, backend, domains });\n const { indexed, failed, skipped } = await follower.reindexPaths(livePaths, (done, total) =>\n opts.onProgress?.({ phase: \"rebuild\", done, total }),\n );\n return { cleared, indexed, failed, skipped };\n}\n"],"mappings":";;;;;AA8EA,eAAsB,oBACpB,MAC+B;CAC/B,MAAM,EAAE,SAAS,UAAU,SAAS,WAAW,uBAAuB;AAqBtE,KAAI,oBAAoB,UAAU,QAAQ,gBAAgB;EACxD,MAAM,WAAW,IAAI,sBAAsB;EAC3C,MAAM,UAAU,IAAI,IAAI,mBAAmB;AAC3C,WAAS,SAAS,SAAS,WAAW,iBAAiB,QAAQ,QAAQ,CAAC;AACxE,UAAQ,eAAe,SAAS;;CAalC,IAAI,UAAU;CACd,MAAM,eAAe,QAAQ,KAAK,MAAM,EAAE,KAAK;CAC/C,MAAM,WAAW,QAAQ,cAAc,MAAM,QAAQ,YAAY,EAAE,cAAc,CAAC,GAAG,EAAE;AACvF,KAAI,SAAS,SAAS,KAAK,QAAQ,cAAc;AAC/C,OAAK,aAAa;GAAE,OAAO;GAAS,MAAM;GAAG,OAAO,SAAS;GAAQ,CAAC;AACtE,YAAU,MAAM,QAAQ,aACtB,SAAS,KAAK,MAAM,EAAE,UAAU,EAChC,EAAE,YAAY,MAAM,CACrB;AACD,OAAK,aAAa;GAAE,OAAO;GAAS,MAAM;GAAS,OAAO,SAAS;GAAQ,CAAC;;CAO9E,MAAM,EAAE,SAAS,QAAQ,YAAY,MADpB,IAAI,cAAc;EAAE;EAAU;EAAS;EAAS,CAAC,CACd,aAAa,YAAY,MAAM,UACjF,KAAK,aAAa;EAAE,OAAO;EAAW;EAAM;EAAO,CAAC,CACrD;AACD,QAAO;EAAE;EAAS;EAAS;EAAQ;EAAS"}
|
|
@@ -0,0 +1,433 @@
|
|
|
1
|
+
const require_fts = require('../text/fts.cjs');
|
|
2
|
+
const require_limits = require('./limits.cjs');
|
|
3
|
+
|
|
4
|
+
//#region src/query/engine.ts
|
|
5
|
+
const QUERY_CURSOR_VERSION = 1;
|
|
6
|
+
const MAX_QUERY_WINDOW = 2e3;
|
|
7
|
+
function stableJson(value) {
|
|
8
|
+
if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "undefined";
|
|
9
|
+
if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`;
|
|
10
|
+
const obj = value;
|
|
11
|
+
return `{${Object.keys(obj).sort().filter((k) => obj[k] !== void 0).map((k) => `${JSON.stringify(k)}:${stableJson(obj[k])}`).join(",")}}`;
|
|
12
|
+
}
|
|
13
|
+
function hashString(input) {
|
|
14
|
+
let h = 2166136261;
|
|
15
|
+
for (let i = 0; i < input.length; i++) {
|
|
16
|
+
h ^= input.charCodeAt(i);
|
|
17
|
+
h = Math.imul(h, 16777619);
|
|
18
|
+
}
|
|
19
|
+
return (h >>> 0).toString(36);
|
|
20
|
+
}
|
|
21
|
+
function encodeCursor(payload) {
|
|
22
|
+
return `q1.${encodeURIComponent(JSON.stringify(payload))}`;
|
|
23
|
+
}
|
|
24
|
+
function decodeCursor(cursor) {
|
|
25
|
+
if (!cursor.startsWith("q1.")) return null;
|
|
26
|
+
try {
|
|
27
|
+
const parsed = JSON.parse(decodeURIComponent(cursor.slice(3)));
|
|
28
|
+
if (parsed.v !== QUERY_CURSOR_VERSION || typeof parsed.offset !== "number" || !Number.isFinite(parsed.offset) || parsed.offset < 0 || Math.floor(parsed.offset) !== parsed.offset || typeof parsed.qh !== "string" || !parsed.qh) return null;
|
|
29
|
+
return parsed;
|
|
30
|
+
} catch {
|
|
31
|
+
return null;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Human-readable label for an anchor filter (issue #1069 debug info) — e.g.
|
|
36
|
+
* `person:[name:alice] & topic:[cooking]`. Static per query (queryAnchors ANDs
|
|
37
|
+
* across groups, ORs within a group's values — see `anchor_intersection`), so
|
|
38
|
+
* every entry the filter returns satisfies the same label; safe to reuse
|
|
39
|
+
* across all of a list's entries rather than recomputing per-entry.
|
|
40
|
+
*/
|
|
41
|
+
function describeAnchorFilter(keys) {
|
|
42
|
+
return keys.map((k) => `${k.type}:[${k.values.join(",")}]`).join(" & ");
|
|
43
|
+
}
|
|
44
|
+
/** Human-readable label for an FTS query (issue #1069 debug info). */
|
|
45
|
+
function describeFtsQuery(text) {
|
|
46
|
+
return `"${text}"`;
|
|
47
|
+
}
|
|
48
|
+
function formatContribution(c) {
|
|
49
|
+
return `${c.source}${c.label ? ` ${c.label}` : ""} rank=${c.rank} score=${c.rawScore.toFixed(3)}`;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Reciprocal Rank Fusion — merge multiple ranked lists into one.
|
|
53
|
+
*
|
|
54
|
+
* Each entry's score = Σ(1 / (k + rank + 1)) across all lists it appears in.
|
|
55
|
+
* Rank-based fusion naturally handles different score scales.
|
|
56
|
+
*
|
|
57
|
+
* @param lists - Ranked lists from different retrieval sources (anchor, FTS, embedding)
|
|
58
|
+
* @param k - Smoothing constant (default 60, standard in literature)
|
|
59
|
+
* @param explain - When true (issue #1069), decorate each result with a
|
|
60
|
+
* composite `explanation` listing every contributing source's rank/score
|
|
61
|
+
* and the final RRF math. Default false: identical output to before this
|
|
62
|
+
* flag existed (byte-for-byte — no `explanation` key at all).
|
|
63
|
+
*/
|
|
64
|
+
function reciprocalRankFusion(lists, k = 60, explain = false) {
|
|
65
|
+
const scores = /* @__PURE__ */ new Map();
|
|
66
|
+
for (const list of lists) for (let rank = 0; rank < list.entries.length; rank++) {
|
|
67
|
+
const entry = list.entries[rank];
|
|
68
|
+
const contribution = 1 / (k + rank + 1);
|
|
69
|
+
const existing = scores.get(entry.entryPath);
|
|
70
|
+
const rrfContribution = {
|
|
71
|
+
source: list.source,
|
|
72
|
+
rank,
|
|
73
|
+
rawScore: entry.score,
|
|
74
|
+
label: list.label
|
|
75
|
+
};
|
|
76
|
+
if (existing) {
|
|
77
|
+
existing.rrfScore += contribution;
|
|
78
|
+
existing.contributions.push(rrfContribution);
|
|
79
|
+
if (rank < existing.bestRank) {
|
|
80
|
+
existing.bestRank = rank;
|
|
81
|
+
existing.bestSource = list.source;
|
|
82
|
+
}
|
|
83
|
+
} else scores.set(entry.entryPath, {
|
|
84
|
+
entryPath: entry.entryPath,
|
|
85
|
+
rrfScore: contribution,
|
|
86
|
+
bestSource: list.source,
|
|
87
|
+
bestRank: rank,
|
|
88
|
+
contributions: [rrfContribution]
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
return Array.from(scores.values()).sort((a, b) => b.rrfScore - a.rrfScore || a.entryPath.localeCompare(b.entryPath)).map(({ entryPath, rrfScore, bestSource, contributions }) => {
|
|
92
|
+
const result = {
|
|
93
|
+
entryPath,
|
|
94
|
+
matchType: bestSource,
|
|
95
|
+
score: rrfScore
|
|
96
|
+
};
|
|
97
|
+
if (explain) result.explanation = `${contributions.map(formatContribution).join("; ")}; RRF=${rrfScore.toFixed(4)}`;
|
|
98
|
+
return result;
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Candidate pool size when `anchorFilter` is on (Epic #932 S2/S3 facet search).
|
|
103
|
+
* The anchor∩text intersection needs more than the final `limit` of text
|
|
104
|
+
* candidates so a facet-matching but lower-text-ranked entry isn't dropped
|
|
105
|
+
* before the filter. Sized for personal (aside-scale) data.
|
|
106
|
+
*/
|
|
107
|
+
const ANCHOR_FILTER_CANDIDATES = 500;
|
|
108
|
+
var QueryEngine = class {
|
|
109
|
+
constructor(backend) {
|
|
110
|
+
this.backend = backend;
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Resolve the `boundDomains` fallback filter for a query: `undefined` when
|
|
114
|
+
* the caller passed an explicit `opts.scope` (their scope wins outright,
|
|
115
|
+
* never overridden — see `QueryInput.boundDomains`), otherwise the bound
|
|
116
|
+
* set (or `undefined` if none/empty, a strict no-op preserving today's
|
|
117
|
+
* "no domain → query everything" behavior for the global `/modules/index`
|
|
118
|
+
* mount and any other caller that never sets `boundDomains`).
|
|
119
|
+
*/
|
|
120
|
+
domainsFilter(input) {
|
|
121
|
+
if (input.opts?.scope) return void 0;
|
|
122
|
+
return input.boundDomains && input.boundDomains.length > 0 ? input.boundDomains : void 0;
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* Applies time-decay to a score based on entry's updatedAt timestamp.
|
|
126
|
+
* Formula: score * e^(-lambda * ageDays)
|
|
127
|
+
*/
|
|
128
|
+
async applyDecay(results, lambda) {
|
|
129
|
+
const now = Date.now();
|
|
130
|
+
const decayed = [];
|
|
131
|
+
for (const r of results) {
|
|
132
|
+
const entry = await this.backend.getEntry(r.entryPath);
|
|
133
|
+
if (!entry) {
|
|
134
|
+
decayed.push(r);
|
|
135
|
+
continue;
|
|
136
|
+
}
|
|
137
|
+
const ageDays = (now - new Date(entry.updatedAt).getTime()) / 864e5;
|
|
138
|
+
decayed.push({
|
|
139
|
+
...r,
|
|
140
|
+
score: r.score * Math.exp(-lambda * ageDays)
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
decayed.sort((a, b) => b.score - a.score);
|
|
144
|
+
return decayed;
|
|
145
|
+
}
|
|
146
|
+
queryHash(input, limit) {
|
|
147
|
+
const opts = input.opts ?? {};
|
|
148
|
+
return hashString(stableJson({
|
|
149
|
+
mode: input.mode,
|
|
150
|
+
ftsMode: input.ftsMode,
|
|
151
|
+
anchors: input.anchors,
|
|
152
|
+
text: input.text,
|
|
153
|
+
embedding: input.embedding,
|
|
154
|
+
boundDomains: input.boundDomains,
|
|
155
|
+
opts: {
|
|
156
|
+
anchorFilter: opts.anchorFilter,
|
|
157
|
+
decay: opts.decay,
|
|
158
|
+
embeddingFloor: opts.embeddingFloor,
|
|
159
|
+
limit,
|
|
160
|
+
metaFilter: opts.metaFilter,
|
|
161
|
+
minConfidence: opts.minConfidence,
|
|
162
|
+
minResults: opts.minResults,
|
|
163
|
+
pathPrefix: opts.pathPrefix,
|
|
164
|
+
scope: opts.scope
|
|
165
|
+
}
|
|
166
|
+
}));
|
|
167
|
+
}
|
|
168
|
+
async queryPage(input) {
|
|
169
|
+
const { mode } = input;
|
|
170
|
+
const limit = require_limits.normalizeQueryLimit(input.opts?.limit);
|
|
171
|
+
if (limit <= 0) return { results: [] };
|
|
172
|
+
const qh = this.queryHash(input, limit);
|
|
173
|
+
const cursor = input.opts?.cursor;
|
|
174
|
+
const decoded = cursor ? decodeCursor(cursor) : null;
|
|
175
|
+
if (cursor && (!decoded || decoded.qh !== qh)) throw new Error("Invalid or stale query cursor");
|
|
176
|
+
const offset = decoded?.offset ?? 0;
|
|
177
|
+
const requestedWindow = offset + limit + 1;
|
|
178
|
+
const windowLimit = Math.min(requestedWindow, MAX_QUERY_WINDOW);
|
|
179
|
+
const truncated = requestedWindow > MAX_QUERY_WINDOW;
|
|
180
|
+
let results;
|
|
181
|
+
switch (mode) {
|
|
182
|
+
case "anchor":
|
|
183
|
+
results = await this.queryAnchor(input, windowLimit);
|
|
184
|
+
break;
|
|
185
|
+
case "fts":
|
|
186
|
+
results = await this.queryFTS(input, windowLimit);
|
|
187
|
+
break;
|
|
188
|
+
case "combined":
|
|
189
|
+
results = await this.queryCombined(input, windowLimit);
|
|
190
|
+
break;
|
|
191
|
+
case "semantic":
|
|
192
|
+
results = await this.querySemantic(input, windowLimit);
|
|
193
|
+
break;
|
|
194
|
+
}
|
|
195
|
+
if (input.opts?.decay && results.length > 0) results = await this.applyDecay(results, input.opts.decay.lambda);
|
|
196
|
+
if (input.opts?.explain && input.anchors?.length && results.length > 0) results = await this.attachAnchorDetails(results, input.anchors);
|
|
197
|
+
const page = results.slice(offset, offset + limit);
|
|
198
|
+
const hasMore = !truncated && results.length > offset + limit;
|
|
199
|
+
const meta = hasMore || truncated ? {
|
|
200
|
+
...hasMore ? { cursor: encodeCursor({
|
|
201
|
+
v: QUERY_CURSOR_VERSION,
|
|
202
|
+
offset: offset + limit,
|
|
203
|
+
qh
|
|
204
|
+
}) } : {},
|
|
205
|
+
hasMore,
|
|
206
|
+
...truncated ? { truncated: true } : {}
|
|
207
|
+
} : void 0;
|
|
208
|
+
return {
|
|
209
|
+
results: page,
|
|
210
|
+
...meta ? { meta } : {}
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
async query(input) {
|
|
214
|
+
return (await this.queryPage(input)).results;
|
|
215
|
+
}
|
|
216
|
+
/**
|
|
217
|
+
* For each result whose `matchType === "anchor"`, fetch the entry's stored
|
|
218
|
+
* anchors and filter to the types the query actually asked for — the
|
|
219
|
+
* concrete "why was this record matched" evidence issue #1069 wants.
|
|
220
|
+
* `queryAnchors`'s `anchor_intersection` ANDs across groups (see
|
|
221
|
+
* `sqlite-backend.ts`), so any entry in the anchor list satisfied every
|
|
222
|
+
* requested type; we don't re-verify individual values here (that would
|
|
223
|
+
* require re-deriving generated keys, which live in `AFSIndex`'s
|
|
224
|
+
* `KeyGeneratorRegistry`, not the engine) — we report the entry's own
|
|
225
|
+
* anchors of the requested type(s) as the matched evidence.
|
|
226
|
+
*/
|
|
227
|
+
async attachAnchorDetails(results, anchors) {
|
|
228
|
+
const types = new Set(anchors.map((k) => k.type));
|
|
229
|
+
const out = [];
|
|
230
|
+
for (const r of results) {
|
|
231
|
+
if (r.matchType !== "anchor") {
|
|
232
|
+
out.push(r);
|
|
233
|
+
continue;
|
|
234
|
+
}
|
|
235
|
+
const matched = (await this.backend.getAnchorData(r.entryPath)).filter((a) => types.has(a.anchorType));
|
|
236
|
+
out.push({
|
|
237
|
+
...r,
|
|
238
|
+
anchors: matched
|
|
239
|
+
});
|
|
240
|
+
}
|
|
241
|
+
return out;
|
|
242
|
+
}
|
|
243
|
+
async queryAnchor(input, limit) {
|
|
244
|
+
if (!input.anchors || input.anchors.length === 0) return [];
|
|
245
|
+
const results = await this.backend.queryAnchors({
|
|
246
|
+
keys: input.anchors,
|
|
247
|
+
scope: input.opts?.scope,
|
|
248
|
+
pathPrefix: input.opts?.pathPrefix,
|
|
249
|
+
domains: this.domainsFilter(input),
|
|
250
|
+
minConfidence: input.opts?.minConfidence,
|
|
251
|
+
limit,
|
|
252
|
+
metaFilter: input.opts?.metaFilter
|
|
253
|
+
});
|
|
254
|
+
const explain = input.opts?.explain === true;
|
|
255
|
+
const label = explain ? describeAnchorFilter(input.anchors) : void 0;
|
|
256
|
+
return results.map((r) => ({
|
|
257
|
+
entryPath: r.entryPath,
|
|
258
|
+
matchType: "anchor",
|
|
259
|
+
score: r.score,
|
|
260
|
+
...explain ? { explanation: `anchor ${label} score=${r.score.toFixed(3)}` } : {}
|
|
261
|
+
}));
|
|
262
|
+
}
|
|
263
|
+
async queryFTS(input, limit) {
|
|
264
|
+
if (!input.text) return [];
|
|
265
|
+
const transformed = require_fts.transformFtsQuery(input.text, input.ftsMode);
|
|
266
|
+
if (!transformed) return [];
|
|
267
|
+
const scope = input.opts?.scope;
|
|
268
|
+
const domain = scope ? scope.replace(/^\/+/, "") : void 0;
|
|
269
|
+
const domains = this.domainsFilter(input);
|
|
270
|
+
const pathPrefix = input.opts?.pathPrefix;
|
|
271
|
+
const results = await this.backend.queryFTS(transformed, limit, input.opts?.metaFilter, {
|
|
272
|
+
...domain ? { domain } : {},
|
|
273
|
+
...scope ? { scope } : {},
|
|
274
|
+
...domains ? { domains } : {},
|
|
275
|
+
...pathPrefix ? { pathPrefix } : {}
|
|
276
|
+
});
|
|
277
|
+
const explain = input.opts?.explain === true;
|
|
278
|
+
const label = explain ? describeFtsQuery(transformed) : void 0;
|
|
279
|
+
return results.map((r) => ({
|
|
280
|
+
entryPath: r.entryPath,
|
|
281
|
+
matchType: "fts",
|
|
282
|
+
score: r.rank,
|
|
283
|
+
...explain ? { explanation: `fts ${label} rank=${r.rank.toFixed(3)}` } : {}
|
|
284
|
+
}));
|
|
285
|
+
}
|
|
286
|
+
/** Collect anchor + FTS ranked lists from backend. */
|
|
287
|
+
async collectLists(input, limit) {
|
|
288
|
+
const lists = [];
|
|
289
|
+
if (input.anchors?.length) {
|
|
290
|
+
const results = await this.backend.queryAnchors({
|
|
291
|
+
keys: input.anchors,
|
|
292
|
+
scope: input.opts?.scope,
|
|
293
|
+
pathPrefix: input.opts?.pathPrefix,
|
|
294
|
+
domains: this.domainsFilter(input),
|
|
295
|
+
minConfidence: input.opts?.minConfidence,
|
|
296
|
+
limit,
|
|
297
|
+
metaFilter: input.opts?.metaFilter
|
|
298
|
+
});
|
|
299
|
+
if (results.length > 0) {
|
|
300
|
+
const explain = input.opts?.explain === true;
|
|
301
|
+
lists.push({
|
|
302
|
+
source: "anchor",
|
|
303
|
+
entries: results.map((r) => ({
|
|
304
|
+
entryPath: r.entryPath,
|
|
305
|
+
score: r.score
|
|
306
|
+
})),
|
|
307
|
+
...explain ? { label: describeAnchorFilter(input.anchors) } : {}
|
|
308
|
+
});
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
if (input.text) {
|
|
312
|
+
const transformedText = require_fts.transformFtsQuery(input.text, input.ftsMode);
|
|
313
|
+
if (transformedText) {
|
|
314
|
+
const scope = input.opts?.scope;
|
|
315
|
+
const domain = scope ? scope.replace(/^\/+/, "") : void 0;
|
|
316
|
+
const domains = this.domainsFilter(input);
|
|
317
|
+
const pathPrefix = input.opts?.pathPrefix;
|
|
318
|
+
const results = await this.backend.queryFTS(transformedText, limit, input.opts?.metaFilter, {
|
|
319
|
+
...domain ? { domain } : {},
|
|
320
|
+
...scope ? { scope } : {},
|
|
321
|
+
...domains ? { domains } : {},
|
|
322
|
+
...pathPrefix ? { pathPrefix } : {}
|
|
323
|
+
});
|
|
324
|
+
if (results.length > 0) {
|
|
325
|
+
const explain = input.opts?.explain === true;
|
|
326
|
+
lists.push({
|
|
327
|
+
source: "fts",
|
|
328
|
+
entries: results.map((r) => ({
|
|
329
|
+
entryPath: r.entryPath,
|
|
330
|
+
score: r.rank
|
|
331
|
+
})),
|
|
332
|
+
...explain ? { label: describeFtsQuery(transformedText) } : {}
|
|
333
|
+
});
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
return lists;
|
|
338
|
+
}
|
|
339
|
+
async queryCombined(input, limit) {
|
|
340
|
+
const anchorFilter = input.opts?.anchorFilter ?? false;
|
|
341
|
+
const lists = await this.collectLists(input, this.candidateLimit(limit, anchorFilter));
|
|
342
|
+
return this.fuseOrFilter(lists, limit, anchorFilter, input.opts?.explain === true);
|
|
343
|
+
}
|
|
344
|
+
async querySemantic(input, limit) {
|
|
345
|
+
const anchorFilter = input.opts?.anchorFilter ?? false;
|
|
346
|
+
const explain = input.opts?.explain === true;
|
|
347
|
+
const candidates = this.candidateLimit(limit, anchorFilter);
|
|
348
|
+
const lists = await this.collectLists(input, candidates);
|
|
349
|
+
if (input.embedding && this.backend.queryVector) {
|
|
350
|
+
const scope = input.opts?.scope;
|
|
351
|
+
const domain = scope ? scope.replace(/^\/+/, "") : void 0;
|
|
352
|
+
const domains = this.domainsFilter(input);
|
|
353
|
+
const pathPrefix = input.opts?.pathPrefix;
|
|
354
|
+
const vectorResults = await this.backend.queryVector(input.embedding, candidates, {
|
|
355
|
+
...domain ? { domain } : {},
|
|
356
|
+
...scope ? { scope } : {},
|
|
357
|
+
...domains ? { domains } : {},
|
|
358
|
+
...pathPrefix ? { pathPrefix } : {}
|
|
359
|
+
});
|
|
360
|
+
if (vectorResults.length > 0) {
|
|
361
|
+
let entries = vectorResults.map((r) => ({
|
|
362
|
+
entryPath: r.entryPath,
|
|
363
|
+
score: r.similarity
|
|
364
|
+
}));
|
|
365
|
+
const floor = input.opts?.embeddingFloor;
|
|
366
|
+
if (floor != null) {
|
|
367
|
+
const explicit = /* @__PURE__ */ new Set();
|
|
368
|
+
for (const l of lists) for (const e of l.entries) explicit.add(e.entryPath);
|
|
369
|
+
entries = entries.filter((e) => explicit.has(e.entryPath) || e.score >= floor);
|
|
370
|
+
}
|
|
371
|
+
if (entries.length > 0) lists.push({
|
|
372
|
+
source: "embedding",
|
|
373
|
+
entries,
|
|
374
|
+
...explain ? { label: "cosine" } : {}
|
|
375
|
+
});
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
return this.fuseOrFilter(lists, limit, anchorFilter, explain);
|
|
379
|
+
}
|
|
380
|
+
/**
|
|
381
|
+
* Over-fetch candidates when anchor-filtering so the anchor∩text intersection
|
|
382
|
+
* has enough to work with — an anchor-matching, text-relevant entry ranked
|
|
383
|
+
* beyond `limit` would otherwise be dropped before the filter runs. (For
|
|
384
|
+
* aside-scale personal data this window comfortably covers a tag/collection's
|
|
385
|
+
* items; a truly-huge facet set could still clip the long tail — a
|
|
386
|
+
* backend-level `WHERE entry IN (facet-set)` is the follow-up for that.)
|
|
387
|
+
*/
|
|
388
|
+
candidateLimit(limit, anchorFilter) {
|
|
389
|
+
return anchorFilter ? Math.max(limit, ANCHOR_FILTER_CANDIDATES) : limit;
|
|
390
|
+
}
|
|
391
|
+
/**
|
|
392
|
+
* Combine ranked lists into the final result. Default = RRF fusion (union of
|
|
393
|
+
* every source). When `anchorFilter` is on AND an anchor list is present, the
|
|
394
|
+
* anchors are a HARD FILTER instead (Epic #932 S2/S3 facet search): with text
|
|
395
|
+
* → the text ranking intersected with the anchor-matching set; facet-only (no
|
|
396
|
+
* text) → the anchor set itself. No anchor list (no anchors / none matched) →
|
|
397
|
+
* plain text-only, so `anchorFilter` never turns a valid text search empty.
|
|
398
|
+
*/
|
|
399
|
+
fuseOrFilter(lists, limit, anchorFilter, explain = false) {
|
|
400
|
+
if (lists.length === 0) return [];
|
|
401
|
+
const flatten = (l) => l.entries.map((e, rank) => ({
|
|
402
|
+
entryPath: e.entryPath,
|
|
403
|
+
matchType: l.source,
|
|
404
|
+
score: e.score,
|
|
405
|
+
...explain ? { explanation: formatContribution({
|
|
406
|
+
source: l.source,
|
|
407
|
+
rank,
|
|
408
|
+
rawScore: e.score,
|
|
409
|
+
label: l.label
|
|
410
|
+
}) } : {}
|
|
411
|
+
}));
|
|
412
|
+
if (anchorFilter) {
|
|
413
|
+
const anchorList = lists.find((l) => l.source === "anchor");
|
|
414
|
+
const textLists = lists.filter((l) => l.source !== "anchor");
|
|
415
|
+
if (anchorList) {
|
|
416
|
+
const allowed = new Set(anchorList.entries.map((e) => e.entryPath));
|
|
417
|
+
if (textLists.length === 0) return flatten(anchorList).slice(0, limit);
|
|
418
|
+
const filtered = (textLists.length === 1 ? flatten(textLists[0]) : reciprocalRankFusion(textLists, 60, explain)).filter((r) => allowed.has(r.entryPath)).slice(0, limit);
|
|
419
|
+
if (explain && anchorList.label) return filtered.map((r) => ({
|
|
420
|
+
...r,
|
|
421
|
+
explanation: `${r.explanation ?? ""}; hard-filtered by anchor ${anchorList.label}`.replace(/^; /, "")
|
|
422
|
+
}));
|
|
423
|
+
return filtered;
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
if (lists.length === 1) return flatten(lists[0]).slice(0, limit);
|
|
427
|
+
return reciprocalRankFusion(lists, 60, explain).slice(0, limit);
|
|
428
|
+
}
|
|
429
|
+
};
|
|
430
|
+
|
|
431
|
+
//#endregion
|
|
432
|
+
exports.QueryEngine = QueryEngine;
|
|
433
|
+
exports.reciprocalRankFusion = reciprocalRankFusion;
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { MatchType, QueryResult } from "../types.cjs";
|
|
2
|
+
|
|
3
|
+
//#region src/query/engine.d.ts
|
|
4
|
+
/** A ranked list from a single retrieval source, pre-sorted by relevance. */
|
|
5
|
+
interface RankedList {
|
|
6
|
+
source: MatchType;
|
|
7
|
+
entries: {
|
|
8
|
+
entryPath: string;
|
|
9
|
+
score: number;
|
|
10
|
+
}[];
|
|
11
|
+
/**
|
|
12
|
+
* Human-readable descriptor of what this list represents (e.g. the anchor
|
|
13
|
+
* filter or the FTS query text) — only built when `explain` is requested
|
|
14
|
+
* (issue #1069). Used to compose `QueryResult.explanation`.
|
|
15
|
+
*/
|
|
16
|
+
label?: string;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Reciprocal Rank Fusion — merge multiple ranked lists into one.
|
|
20
|
+
*
|
|
21
|
+
* Each entry's score = Σ(1 / (k + rank + 1)) across all lists it appears in.
|
|
22
|
+
* Rank-based fusion naturally handles different score scales.
|
|
23
|
+
*
|
|
24
|
+
* @param lists - Ranked lists from different retrieval sources (anchor, FTS, embedding)
|
|
25
|
+
* @param k - Smoothing constant (default 60, standard in literature)
|
|
26
|
+
* @param explain - When true (issue #1069), decorate each result with a
|
|
27
|
+
* composite `explanation` listing every contributing source's rank/score
|
|
28
|
+
* and the final RRF math. Default false: identical output to before this
|
|
29
|
+
* flag existed (byte-for-byte — no `explanation` key at all).
|
|
30
|
+
*/
|
|
31
|
+
declare function reciprocalRankFusion(lists: RankedList[], k?: number, explain?: boolean): QueryResult[];
|
|
32
|
+
//#endregion
|
|
33
|
+
export { reciprocalRankFusion };
|
|
34
|
+
//# sourceMappingURL=engine.d.cts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"engine.d.cts","names":[],"sources":["../../src/query/engine.ts"],"mappings":";;;;UAoHU,UAAA;EACR,MAAA,EAAQ,SAAA;EACR,OAAA;IAAW,SAAA;IAAmB,KAAA;EAAA;;;;;;EAM9B,KAAA;AAAA;;;;;;;;;;;;;;iBA4Bc,oBAAA,CAAqB,KAAA,EAAO,UAAA,IAAc,CAAA,WAAQ,OAAA,aAAkB,WAAA"}
|