@functional-systems/srcx 0.1.0-dev.1 → 0.1.0-dev.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +56 -1
- package/DESIGN.md +102 -34
- package/README.md +163 -12
- package/RELEASING.md +17 -8
- package/VALIDATION.md +325 -19
- package/dist/build.d.ts +73 -1
- package/dist/build.js +122 -21
- package/dist/build.js.map +1 -1
- package/dist/chunk.d.ts +2 -1
- package/dist/chunk.js +10 -9
- package/dist/chunk.js.map +1 -1
- package/dist/cli.js +55 -18
- package/dist/cli.js.map +1 -1
- package/dist/publish.d.ts +18 -0
- package/dist/publish.js +154 -19
- package/dist/publish.js.map +1 -1
- package/dist/releases.js +21 -3
- package/dist/releases.js.map +1 -1
- package/dist/remote.js +38 -9
- package/dist/remote.js.map +1 -1
- package/dist/repository.d.ts +4 -1
- package/dist/repository.js +15 -11
- package/dist/repository.js.map +1 -1
- package/dist/search.d.ts +7 -2
- package/dist/search.js +24 -4
- package/dist/search.js.map +1 -1
- package/package.json +13 -5
- package/src/build.ts +166 -24
- package/src/chunk.ts +12 -7
- package/src/cli.ts +105 -22
- package/src/publish.ts +222 -20
- package/src/releases.ts +36 -3
- package/src/remote.ts +59 -13
- package/src/repository.ts +20 -11
- package/src/search.ts +37 -8
package/dist/repository.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { join } from "node:path";
|
|
3
3
|
import { atomic, hash, invariant, optionalJson, PURPOSE, } from "./common.js";
|
|
4
|
-
import { PRESET,
|
|
4
|
+
import { PRESET, indexConfigs, supportedPreset } from "./build.js";
|
|
5
5
|
import { identity } from "./git.js";
|
|
6
6
|
import { one, branch } from "./remote.js";
|
|
7
7
|
import { exclusive } from "./publish.js";
|
|
@@ -18,13 +18,13 @@ export function collectionName(key, name, configHash) {
|
|
|
18
18
|
return `code-${slug}-${hash(["collection-v1", key, configHash]).slice(0, 16)}`;
|
|
19
19
|
}
|
|
20
20
|
/** LambdaDB adds its built-in keyword id index to collection metadata. */
|
|
21
|
-
export function matchesIndexSchema(actual) {
|
|
21
|
+
export function matchesIndexSchema(actual, preset = PRESET) {
|
|
22
22
|
if (!actual || typeof actual !== "object" || Array.isArray(actual))
|
|
23
23
|
return false;
|
|
24
24
|
const { id, ...fields } = actual;
|
|
25
25
|
if (id !== undefined && hash(id) !== hash({ type: "keyword" }))
|
|
26
26
|
return false;
|
|
27
|
-
return hash(fields) === hash(
|
|
27
|
+
return hash(fields) === hash(indexConfigs(preset));
|
|
28
28
|
}
|
|
29
29
|
function descriptor(r) {
|
|
30
30
|
return {
|
|
@@ -37,24 +37,25 @@ function descriptor(r) {
|
|
|
37
37
|
repoId: r.repoId,
|
|
38
38
|
indexId: r.indexId,
|
|
39
39
|
name: r.name,
|
|
40
|
-
preset: PRESET,
|
|
40
|
+
preset: r.preset ?? PRESET,
|
|
41
41
|
initialization: "ready",
|
|
42
42
|
};
|
|
43
43
|
}
|
|
44
44
|
function decode(doc, c) {
|
|
45
45
|
invariant(doc?.role === "repository" &&
|
|
46
46
|
doc.initialization === "ready" &&
|
|
47
|
-
doc.
|
|
48
|
-
|
|
47
|
+
supportedPreset(doc.preset) &&
|
|
48
|
+
doc.configHash === hash(doc.preset), "Repository is partially initialized or uses an unsupported preset.");
|
|
49
49
|
invariant(doc.indexId === c.tags["index-id"] &&
|
|
50
50
|
doc.configHash === c.tags["config-hash"] &&
|
|
51
51
|
c.tags.purpose === PURPOSE, "Repository descriptor and Collection labels disagree.");
|
|
52
|
-
invariant(matchesIndexSchema(c.indexConfigs), "Collection schema differs from the pinned preset.");
|
|
52
|
+
invariant(matchesIndexSchema(c.indexConfigs, doc.preset), "Collection schema differs from the pinned preset.");
|
|
53
53
|
invariant(typeof doc.repoKey === "string" &&
|
|
54
54
|
typeof doc.repoId === "string" &&
|
|
55
55
|
typeof doc.indexId === "string" &&
|
|
56
56
|
typeof doc.name === "string", "Malformed repository descriptor.");
|
|
57
57
|
return {
|
|
58
|
+
preset: doc.preset,
|
|
58
59
|
collection: c.collectionName,
|
|
59
60
|
repoKey: doc.repoKey,
|
|
60
61
|
repoId: doc.repoId,
|
|
@@ -91,13 +92,15 @@ export async function selectRepository(remote, selector) {
|
|
|
91
92
|
r.repoKey.split("/").slice(1).join("/") === selector ||
|
|
92
93
|
r.collection === selector);
|
|
93
94
|
invariant(selected.length === 1, selected.length
|
|
94
|
-
? "Ambiguous repository; use its
|
|
95
|
+
? "Ambiguous repository; use its exact Collection name (presets have separate Collections)."
|
|
95
96
|
: `Repository not found${partial.length ? "; partially initialized collections exist (see repo list)" : ""}.`);
|
|
96
97
|
return selected[0];
|
|
97
98
|
}
|
|
98
99
|
export async function register(remote, args) {
|
|
99
100
|
const source = await identity(args.path, args.remote);
|
|
100
|
-
const
|
|
101
|
+
const preset = args.preset ?? PRESET;
|
|
102
|
+
invariant(supportedPreset(preset), "Unsupported repository preset.");
|
|
103
|
+
const configHash = hash(preset), collection = collectionName(source.key, source.name, configHash), state = destination(remote.settings, collection);
|
|
101
104
|
return exclusive(state, async () => {
|
|
102
105
|
const initPath = join(state, "initialization.json");
|
|
103
106
|
let intent = await optionalJson(initPath);
|
|
@@ -127,6 +130,7 @@ export async function register(remote, args) {
|
|
|
127
130
|
invariant(description.length <= 255, "Description exceeds 255 characters.");
|
|
128
131
|
const indexId = randomUUID();
|
|
129
132
|
intent = {
|
|
133
|
+
preset,
|
|
130
134
|
repoKey: source.key,
|
|
131
135
|
name: source.name,
|
|
132
136
|
repoId: randomUUID(),
|
|
@@ -149,9 +153,9 @@ export async function register(remote, args) {
|
|
|
149
153
|
invariant(intent.repoKey === source.key && intent.configHash === configHash, "Provisioning journal identity mismatch.");
|
|
150
154
|
if (existing)
|
|
151
155
|
invariant(hash(existing.tags) === hash(intent.tags) &&
|
|
152
|
-
matchesIndexSchema(existing.indexConfigs), "Existing Collection differs from provisioning journal.");
|
|
156
|
+
matchesIndexSchema(existing.indexConfigs, preset), "Existing Collection differs from provisioning journal.");
|
|
153
157
|
else
|
|
154
|
-
await remote.create(collection, JSON.parse(JSON.stringify(
|
|
158
|
+
await remote.create(collection, JSON.parse(JSON.stringify(indexConfigs(preset))), intent.description, intent.tags);
|
|
155
159
|
const store = remote.store(collection);
|
|
156
160
|
const branches = await store.branches();
|
|
157
161
|
const empty = branches.find((b) => b.name === "checkpoint-empty");
|
package/dist/repository.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"repository.js","sourceRoot":"","sources":["../src/repository.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EACL,MAAM,EACN,IAAI,EACJ,SAAS,EACT,YAAY,EACZ,OAAO,GAER,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,MAAM,EAAE,
|
|
1
|
+
{"version":3,"file":"repository.js","sourceRoot":"","sources":["../src/repository.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EACL,MAAM,EACN,IAAI,EACJ,SAAS,EACT,YAAY,EACZ,OAAO,GAER,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,eAAe,EAAe,MAAM,YAAY,CAAC;AAChF,OAAO,EAAE,QAAQ,EAAiB,MAAM,UAAU,CAAC;AACnD,OAAO,EAAgB,GAAG,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACxD,OAAO,EAAE,SAAS,EAAgB,MAAM,cAAc,CAAC;AACvD,OAAO,EAAE,SAAS,EAAiB,MAAM,eAAe,CAAC;AASzD,MAAM,UAAU,WAAW,CAAC,CAAW,EAAE,UAAkB;IACzD,OAAO,IAAI,CACT,SAAS,EAAE,EACX,cAAc,EACd,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAC7B,UAAU,CACX,CAAC;AACJ,CAAC;AACD,MAAM,UAAU,cAAc,CAC5B,GAAW,EACX,IAAY,EACZ,UAAkB;IAElB,MAAM,IAAI,GACR,IAAI;SACD,WAAW,EAAE;SACb,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;SACvB,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC;IAC5B,OAAO,QAAQ,IAAI,IAAI,IAAI,CAAC,CAAC,eAAe,EAAE,GAAG,EAAE,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;AACjF,CAAC;AACD,0EAA0E;AAC1E,MAAM,UAAU,kBAAkB,CAChC,MAAe,EACf,SAAiB,MAAM;IAEvB,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;QAChE,OAAO,KAAK,CAAC;IACf,MAAM,EAAE,EAAE,EAAE,GAAG,MAAM,EAAE,GAAG,MAAiC,CAAC;IAC5D,IAAI,EAAE,KAAK,SAAS,IAAI,IAAI,CAAC,EAAE,CAAC,KAAK,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7E,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;AACrD,CAAC;AACD,SAAS,UAAU,CAAC,CAAa;IAC/B,OAAO;QACL,EAAE,EAAE,UAAU;QACd,IAAI,EAAE,UAAU;QAChB,IAAI,EAAE,YAAY;QAClB,aAAa,EAAE,CAAC;QAChB,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,MAAM,EAAE,CAAC,CAAC,MAAM,IAAI,MAAM;QAC1B,cAAc,EAAE,OAAO;KACxB,CAAC;AACJ,CAAC;AACD,SAAS,MAAM,CACb,GAAoB,EACpB,CAKC;IAED,SAAS,CACP,GAAG,EAAE,IAAI,KAAK,YAAY;QACxB,GAAG,CAAC,cAAc,KAAK,OAAO;QAC9B,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC;QAC3B,GAAG,CAAC,UAAU,KAAK,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EACrC,oEAAoE,CACrE,CAAC;IACF,SAAS,CACP,GAAG,CAAC,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC;QAChC,GAAG,CAAC,UAAU,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC;QACxC,CAAC,CAAC,IAAI,CAAC,OAAO,KAAK,OAAO,EAC5B,uDAAuD,CACxD,CAAC;IACF,SAAS,CACP,kBAAkB,CAAC,CAAC,CAAC,YAAY,EAAE,GAAG,CAAC,MAAgB,CAAC,EACxD,mDAAmD,CACpD,CAAC;IACF,SAAS,CACP,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ;QAC7B,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ;QAC9B,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ;QAC/B,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,EAC9B,kCAAkC,CACnC,CAAC;IACF,OAAO;QACL,MAAM,EAAE,GAAG,CAAC,MAAgB;QAC5B,UAAU,EAAE,CAAC,CAAC,cAAc;QAC5B,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC;QAClC,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,IAAI,EAAE,CAAC,CAAC,IAAI;KACb,CAAC;AACJ,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,MAAoB;IAIjD,MAAM,YAAY,GAAiB,EAAE,CAAC;IACtC,MAAM,OAAO,GAA6C,EAAE,CAAC;IAC7D,KAAK,MAAM,CAAC,IAAI,MAAM,MAAM,CAAC,WAAW,EAAE,EAAE,CAAC;QAC3C,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,KAAK,OAAO;YAAE,SAAS;QACzC,MAAM,GAAG,GAAG,MAAM,GAAG,CACnB,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,EAC9B,MAAM,CAAC,MAAM,CAAC,EACd,UAAU,EACV,IAAI,CACL,CAAC;QACF,IAAI,CAAC;YACH,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QACpC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,IAAI,CAAC;gBACX,UAAU,EAAE,CAAC,CAAC,cAAc;gBAC5B,MAAM,EAAE,uDAAuD;aAChE,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC;AACnC,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,MAAoB,EACpB,QAAgB;IAEhB,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAClC,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,IAAI,KAAK,QAAQ;QACnB,CAAC,CAAC,OAAO,KAAK,QAAQ;QACtB,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,QAAQ;QACpD,CAAC,CAAC,UAAU,KAAK,QAAQ,CAC5B,CAAC;IACF,SAAS,CACP,QAAQ,CAAC,MAAM,KAAK,CAAC,EACrB,QAAQ,CAAC,MAAM;QACb,CAAC,CAAC,0FAA0F;QAC5F,CAAC,CAAC,uBAAuB,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,2DAA2D,CAAC,CAAC,CAAC,EAAE,GAAG,CAChH,CAAC;IACF,OAAO,QAAQ,CAAC,CAAC,CAAE,CAAC;AACtB,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,MAAoB,EACpB,IAMC;IAED,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC;IACrC,SAAS,CAAC,eAAe,CAAC,MAAM,CAAC,EAAE,gCAAgC,CAAC,CAAC;IACrE,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,EAC7B,UAAU,GAAG,cAAc,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,EAAE,UAAU,CAAC,EAChE,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IACnD,OAAO,SAAS,CAAC,KAAK,EAAE,KAAK,IAAI,EAAE;QACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,EAAE,qBAAqB,CAAC,CAAC;QACpD,IAAI,MAAM,GAAG,MAAM,YAAY,CAAa,QAAQ,CAAC,CAAC;QACtD,MAAM,QAAQ,GAAG,CAAC,MAAM,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,IAAI,CAChD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,KAAK,UAAU,CACvC,CAAC;QACF,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,CAAC,GAAG,MAAM,GAAG,CACjB,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,EACxB,MAAM,CAAC,MAAM,CAAC,EACd,UAAU,EACV,IAAI,CACL,CAAC;YACF,IAAI,CAAC,EAAE,CAAC;gBACN,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;gBACnC,SAAS,CAAC,MAAM,CAAC,OAAO,KAAK,MAAM,CAAC,GAAG,EAAE,4BAA4B,CAAC,CAAC;gBACvE,MAAM,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,iBAAiB,CAAC,EAAE,MAAM,CAAC,CAAC;gBACrD,OAAO,MAAM,CAAC;YAChB,CAAC;QACH,CAAC;QACD,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,SAAS,CACP,CAAC,QAAQ,EACT,4FAA4F,CAC7F,CAAC;YACF,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAG,CAAC,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,aAAa,CAAC,CAAC;YACtE,SAAS,CACP,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,IAAI,CAAC;gBAC7B,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EACxD,kDAAkD,CACnD,CAAC;YACF,SAAS,CACP,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,CAC1B,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CACT,wBAAwB,CAAC,IAAI,CAAC,CAAC,CAAC;gBAChC,CAAC,CAAC,MAAM,IAAI,CAAC;gBACb,CAAC,CAAC,MAAM,IAAI,GAAG;gBACf,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;gBACnB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CACnB,EACD,uBAAuB,CACxB,CAAC;YACF,MAAM,WAAW,GACf,IAAI,CAAC,WAAW;gBAChB,wCAAwC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,8BAA8B,CAAC;YACjG,SAAS,CACP,WAAW,CAAC,MAAM,IAAI,GAAG,EACzB,qCAAqC,CACtC,CAAC;YACF,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC;YAC7B,MAAM,GAAG;gBACP,MAAM;gBACN,OAAO,EAAE,MAAM,CAAC,GAAG;gBACnB,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,MAAM,EAAE,UAAU,EAAE;gBACpB,OAAO;gBACP,UAAU;gBACV,UAAU;gBACV,WAAW;gBACX,IAAI,EAAE;oBACJ,OAAO,EAAE,OAAO;oBAChB,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC;wBACvD,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,GAAG,EAAE;wBAC5B,CAAC,CAAC,EAAE,CAAC;oBACP,UAAU,EAAE,OAAO;oBACnB,aAAa,EAAE,UAAU;oBACzB,GAAG,MAAM;iBACV;aACF,CAAC;YACF,MAAM,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACjC,CAAC;QACD,SAAS,CACP,MAAM,CAAC,OAAO,KAAK,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,UAAU,KAAK,UAAU,EACjE,yCAAyC,CAC1C,CAAC;QACF,IAAI,QAAQ;YACV,SAAS,CACP,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;gBACvC,kBAAkB,CAAC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC,EACnD,wDAAwD,CACzD,CAAC;;YAEF,MAAM,MAAM,CAAC,MAAM,CACjB,UAAU,EACV,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAG9C,EACD,MAAM,CAAC,WAAW,EAClB,MAAM,CAAC,IAAI,CACZ,CAAC;QACJ,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACvC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,kBAAkB,CAAC,CAAC;QAClE,IAAI,KAAK;YACP,SAAS,CACP,KAAK,CAAC,UAAU,KAAK,IAAI,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM,EACpD,qCAAqC,CACtC,CAAC;aACC,CAAC;YACJ,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;YACrD,SAAS,CACP,IAAI,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI,EAChC,yDAAyD,CAC1D,CAAC;YACF,MAAM,KAAK,CAAC,MAAM,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAC;QACjD,CAAC;QACD,MAAM,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACjD,MAAM,CAAC,GAAG,CAAC,MAAM,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,IAAI,CACzC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,KAAK,UAAU,CACvC,CAAC;QACF,SAAS,CAAC,CAAC,EAAE,sDAAsD,CAAC,CAAC;QACrE,MAAM,MAAM,GAAG,MAAM,CACnB,MAAM,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,EAClD,CAAC,CACF,CAAC;QACF,MAAM,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,iBAAiB,CAAC,EAAE,MAAM,CAAC,CAAC;QACrD,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC,CAAC;AACL,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,CAAW,EACX,CAAa;IAEb,MAAM,KAAK,GAAG,MAAM,YAAY,CAC9B,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC,EAAE,iBAAiB,CAAC,CACtD,CAAC;IACF,SAAS,CACP,KAAK,EACL,sEAAsE,CACvE,CAAC;IACF,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACzD,SAAS,CACP,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC,OAAO,EACzB,wDAAwD,CACzD,CAAC;IACF,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
package/dist/search.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { type CollectionStore } from "./remote.js";
|
|
|
2
2
|
import { type Settings } from "./settings.js";
|
|
3
3
|
import type { Repository } from "./repository.js";
|
|
4
4
|
import type { Published } from "./publish.js";
|
|
5
|
-
import type
|
|
5
|
+
import { type InventoryItem } from "./build.js";
|
|
6
6
|
export type Handle = {
|
|
7
7
|
id: string;
|
|
8
8
|
endpoint: string;
|
|
@@ -22,10 +22,15 @@ export declare function lexicalQuery(query: string, filters?: {
|
|
|
22
22
|
path?: string;
|
|
23
23
|
language?: string;
|
|
24
24
|
}): Record<string, unknown>;
|
|
25
|
+
export type SearchMode = "lexical" | "semantic" | "hybrid";
|
|
26
|
+
export declare function retrievalQuery(query: string, size: number, filters: {
|
|
27
|
+
path?: string;
|
|
28
|
+
language?: string;
|
|
29
|
+
}, mode: SearchMode): Record<string, unknown>;
|
|
25
30
|
export declare function search(store: CollectionStore, s: Settings, r: Repository, v: Published, query: string, size?: number, filters?: {
|
|
26
31
|
path?: string;
|
|
27
32
|
language?: string;
|
|
28
|
-
}): Promise<unknown[]>;
|
|
33
|
+
}, mode?: SearchMode): Promise<unknown[]>;
|
|
29
34
|
export declare function loadHandle(id: string): Promise<Handle>;
|
|
30
35
|
export declare function directHandle(store: CollectionStore, s: Settings, r: Repository, v: Published, path: string): Promise<Handle>;
|
|
31
36
|
export declare function readHandle(store: CollectionStore, s: Settings, h: Handle, options?: {
|
package/dist/search.js
CHANGED
|
@@ -3,6 +3,7 @@ import { randomUUID } from "node:crypto";
|
|
|
3
3
|
import { atomic, hash, invariant, json, lineAt } from "./common.js";
|
|
4
4
|
import { one, tag } from "./remote.js";
|
|
5
5
|
import { stateRoot } from "./settings.js";
|
|
6
|
+
import { PRESET, recordHash } from "./build.js";
|
|
6
7
|
export async function inventoryAt(store, r, v) {
|
|
7
8
|
const current = (await store.tags()).find((t) => t.name === v.tagName);
|
|
8
9
|
invariant(current?.snapshotId === v.snapshotId, "Pinned Tag was deleted or recreated.");
|
|
@@ -45,12 +46,31 @@ export function lexicalQuery(query, filters = {}) {
|
|
|
45
46
|
],
|
|
46
47
|
};
|
|
47
48
|
}
|
|
48
|
-
export
|
|
49
|
+
export function retrievalQuery(query, size, filters, mode) {
|
|
50
|
+
invariant(["lexical", "semantic", "hybrid"].includes(mode), "Unknown search mode.");
|
|
51
|
+
const lexical = lexicalQuery(query, filters);
|
|
52
|
+
if (mode === "lexical")
|
|
53
|
+
return lexical;
|
|
54
|
+
const semantic = {
|
|
55
|
+
knn: {
|
|
56
|
+
field: "embedding",
|
|
57
|
+
queryText: query,
|
|
58
|
+
k: size,
|
|
59
|
+
filter: {
|
|
60
|
+
bool: lexical.bool.filter((c) => c.occur === "filter"),
|
|
61
|
+
},
|
|
62
|
+
},
|
|
63
|
+
};
|
|
64
|
+
return mode === "semantic" ? semantic : { rrf: [lexical, semantic] };
|
|
65
|
+
}
|
|
66
|
+
export async function search(store, s, r, v, query, size = 10, filters = {}, mode = "lexical") {
|
|
49
67
|
invariant(Number.isInteger(size) && size > 0 && size <= 100, "Limit must be an integer from 1 to 100.");
|
|
68
|
+
invariant(mode === "lexical" || r.preset?.embedding?.managed, "Semantic/hybrid search requires a managed embedding Collection; register with --embedding text-embedding-3-small or text-embedding-3-large.");
|
|
69
|
+
const request = retrievalQuery(query, size, filters, mode);
|
|
50
70
|
const entries = await inventoryAt(store, r, v);
|
|
51
71
|
const files = new Map(entries.filter((e) => e.status === "included").map((e) => [e.fileId, e]));
|
|
52
72
|
const result = [];
|
|
53
|
-
for (const hit of await store.query(v.tagName,
|
|
73
|
+
for (const hit of await store.query(v.tagName, request, size)) {
|
|
54
74
|
const d = hit.doc, e = files.get(d.fileId);
|
|
55
75
|
invariant(d.kind === "chunk" &&
|
|
56
76
|
d.configHash === r.configHash &&
|
|
@@ -68,7 +88,7 @@ export async function search(store, s, r, v, query, size = 10, filters = {}) {
|
|
|
68
88
|
path: String(d.path),
|
|
69
89
|
contentHash: String(d.contentHash),
|
|
70
90
|
chunkId: d.id,
|
|
71
|
-
chunkHash:
|
|
91
|
+
chunkHash: recordHash(d, r.preset ?? PRESET),
|
|
72
92
|
startLine: Number(d.startLine),
|
|
73
93
|
endLine: Number(d.endLine),
|
|
74
94
|
};
|
|
@@ -132,7 +152,7 @@ export async function readHandle(store, s, h, options = {}) {
|
|
|
132
152
|
if (h.chunkId) {
|
|
133
153
|
const d = await one(store, tag(v.tagName), h.chunkId);
|
|
134
154
|
invariant(d &&
|
|
135
|
-
|
|
155
|
+
recordHash(d, h.repository.preset ?? PRESET) === h.chunkHash &&
|
|
136
156
|
d.fileId === h.fileId &&
|
|
137
157
|
d.contentHash === h.contentHash, "Pinned chunk has changed or is missing.");
|
|
138
158
|
const a = Number(d.startByte), z = Number(d.endByte);
|
package/dist/search.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search.js","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAY,MAAM,aAAa,CAAC;AAC9E,OAAO,EAAE,GAAG,EAAE,GAAG,EAAwB,MAAM,aAAa,CAAC;AAC7D,OAAO,EAAE,SAAS,EAAiB,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"search.js","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAY,MAAM,aAAa,CAAC;AAC9E,OAAO,EAAE,GAAG,EAAE,GAAG,EAAwB,MAAM,aAAa,CAAC;AAC7D,OAAO,EAAE,SAAS,EAAiB,MAAM,eAAe,CAAC;AAGzD,OAAO,EAAE,MAAM,EAAE,UAAU,EAAsB,MAAM,YAAY,CAAC;AAepE,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,KAAsB,EACtB,CAAa,EACb,CAAY;IAEZ,MAAM,OAAO,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;IACvE,SAAS,CACP,OAAO,EAAE,UAAU,KAAK,CAAC,CAAC,UAAU,EACpC,sCAAsC,CACvC,CAAC;IACF,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,cAAc,CAAC,CAAC;IAC9D,SAAS,CACP,IAAI,EAAE,MAAM,KAAK,CAAC,CAAC,MAAM;QACvB,IAAI,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO;QAC1B,IAAI,CAAC,UAAU,KAAK,CAAC,CAAC,UAAU;QAChC,IAAI,CAAC,SAAS,KAAK,CAAC,CAAC,SAAS;QAC9B,IAAI,CAAC,WAAW,KAAK,CAAC,CAAC,WAAW;QAClC,IAAI,CAAC,aAAa,KAAK,CAAC,CAAC,aAAa,EACxC,oCAAoC,CACrC,CAAC;IACF,SAAS,CACP,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,EACpC,sCAAsC,CACvC,CAAC;IACF,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,KAAK,MAAM,CAAC,CAAC,EAAE,EAAE,CAAC,IAAK,IAAI,CAAC,gBAA6B,CAAC,OAAO,EAAE,EAAE,CAAC;QACpE,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;QAClD,SAAS,CACP,IAAI,EAAE,IAAI,KAAK,WAAW;YACxB,IAAI,CAAC,UAAU,KAAK,CAAC,CAAC,UAAU;YAChC,IAAI,CAAC,WAAW,KAAK,CAAC;YACtB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;YAC3B,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,QAAQ,EACtC,yBAAyB,CAC1B,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,GAAI,IAAI,CAAC,OAA2B,CAAC,CAAC;IACrD,CAAC;IACD,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,aAAa,EAAE,0BAA0B,CAAC,CAAC;IAC5E,OAAO,OAAO,CAAC;AACjB,CAAC;AACD,MAAM,UAAU,YAAY,CAC1B,KAAa,EACb,UAAgD,EAAE;IAElD,SAAS,CACP,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,EAC/C,uCAAuC,CACxC,CAAC;IACF,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,KAAa,EAAE,EAAE,CAAC,CAAC;QACjD,WAAW,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE;QACpE,KAAK,EAAE,QAAQ;KAChB,CAAC,CAAC;IACH,OAAO;QACL,IAAI,EAAE;YACJ,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC;YACxB;gBACE,WAAW,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,IAAI,EAAE;gBACpE,KAAK,EAAE,MAAM;aACd;YACD,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACxD,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;SACrE;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,cAAc,CAC5B,KAAa,EACb,IAAY,EACZ,OAA6C,EAC7C,IAAgB;IAEhB,SAAS,CACP,CAAC,SAAS,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAChD,sBAAsB,CACvB,CAAC;IACF,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC7C,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,OAAO,CAAC;IACvC,MAAM,QAAQ,GAAG;QACf,GAAG,EAAE;YACH,KAAK,EAAE,WAAW;YAClB,SAAS,EAAE,KAAK;YAChB,CAAC,EAAE,IAAI;YACP,MAAM,EAAE;gBACN,IAAI,EAAG,OAAO,CAAC,IAA4B,CAAC,MAAM,CAChD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,QAAQ,CAC5B;aACF;SACF;KACF,CAAC;IACF,OAAO,IAAI,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,CAAC;AACvE,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,MAAM,CAC1B,KAAsB,EACtB,CAAW,EACX,CAAa,EACb,CAAY,EACZ,KAAa,EACb,IAAI,GAAG,EAAE,EACT,UAAgD,EAAE,EAClD,OAAmB,SAAS;IAE5B,SAAS,CACP,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,IAAI,IAAI,GAAG,EACjD,yCAAyC,CAC1C,CAAC;IACF,SAAS,CACP,IAAI,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,EAClD,6IAA6I,CAC9I,CAAC;IACF,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IAC3D,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/C,MAAM,KAAK,GAAG,IAAI,GAAG,CACnB,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CACzE,CAAC;IACF,MAAM,MAAM,GAAG,EAAE,CAAC;IAClB,KAAK,MAAM,GAAG,IAAI,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC;QAC9D,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,EACf,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,MAAgB,CAAC,CAAC;QACpC,SAAS,CACP,CAAC,CAAC,IAAI,KAAK,OAAO;YAChB,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,UAAU;YAC7B,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3B,CAAC,CAAC,WAAW,KAAK,CAAC,CAAC,WAAW;YAC/B,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,EACnB,qDAAqD,CACtD,CAAC;QACF,MAAM,EAAE,GAAG,UAAU,EAAE,CAAC;QACxB,MAAM,MAAM,GAAW;YACrB,EAAE;YACF,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,CAAC;YACV,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;YACxB,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;YACpB,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC;YAClC,OAAO,EAAE,CAAC,CAAC,EAAE;YACb,SAAS,EAAE,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,IAAI,MAAM,CAAC;YAC5C,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;YAC9B,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;SAC3B,CAAC;QACF,sEAAsE;QACtE,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QACpD,MAAM,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,EAAE,GAAG,OAAO,CAAC,EAAE,MAAM,CAAC,CAAC;QACjE,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,EAAE;YACZ,UAAU,EAAE,CAAC,CAAC,OAAO;YACrB,SAAS,EAAE,CAAC,CAAC,SAAS;YACtB,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,UAAU,EAAE,CAAC,CAAC,UAAU;YACxB,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,SAAS,EAAE,CAAC,CAAC,SAAS;YACtB,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;YAC1C,QAAQ,EAAE,QAAQ,CAAC,QAAQ;SAC5B,CAAC,CAAC;IACL,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,EAAU;IACzC,SAAS,CAAC,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,2BAA2B,CAAC,CAAC;IACnE,MAAM,CAAC,GAAG,MAAM,IAAI,CAAS,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC;IACzE,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,yBAAyB,CAAC,CAAC;IAClD,OAAO,CAAC,CAAC;AACX,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAsB,EACtB,CAAW,EACX,CAAa,EACb,CAAY,EACZ,IAAY;IAEZ,MAAM,CAAC,GAAG,CAAC,MAAM,WAAW,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;IACxE,SAAS,CACP,CAAC,EAAE,MAAM,KAAK,UAAU,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,WAAW,EACrD,sBAAsB,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,CAC1D,CAAC;IACF,OAAO;QACL,EAAE,EAAE,QAAQ;QACZ,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,UAAU,EAAE,CAAC;QACb,OAAO,EAAE,CAAC;QACV,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,IAAI;QACJ,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,SAAS,EAAE,CAAC;QACZ,OAAO,EAAE,CAAC;KACX,CAAC;AACJ,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,KAAsB,EACtB,CAAW,EACX,CAAS,EACT,UAII,EAAE;IAEN,SAAS,CACP,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,EACpD,qFAAqF,CACtF,CAAC;IACF,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,EACjB,OAAO,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;IACnE,SAAS,CACP,OAAO,EAAE,UAAU,KAAK,CAAC,CAAC,UAAU,EACpC,8CAA8C,CAC/C,CAAC;IACF,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;IACxD,SAAS,CACP,IAAI,EAAE,IAAI,KAAK,MAAM;QACnB,IAAI,CAAC,UAAU,KAAK,CAAC,CAAC,UAAU,CAAC,UAAU;QAC3C,IAAI,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI;QACpB,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ;QACnC,IAAI,CAAC,WAAW,KAAK,CAAC,CAAC,WAAW;QAClC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,CAAC,WAAW,EACtD,wDAAwD,CACzD,CAAC;IACF,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACzC,MAAM,KAAK,GACT,GAAG,CAAC,MAAM,KAAK,CAAC;QACd,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5D,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;QACd,MAAM,CAAC,GAAG,MAAM,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;QACtD,SAAS,CACP,CAAC;YACC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS;YAC5D,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;YACrB,CAAC,CAAC,WAAW,KAAK,CAAC,CAAC,WAAW,EACjC,yCAAyC,CAC1C,CAAC;QACF,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,EAC3B,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACxB,SAAS,CACP,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;YACjB,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;YACnB,CAAC,IAAI,CAAC;YACN,CAAC,GAAG,CAAC;YACL,CAAC,IAAI,GAAG,CAAC,MAAM;YACf,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,SAAS;YAC9B,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,EAClC,uBAAuB,CACxB,CAAC;IACJ,CAAC;IACD,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC;IACrC,SAAS,CACP,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,OAAO,IAAI,CAAC,EACzC,wCAAwC,CACzC,CAAC;IACF,IAAI,KAAK,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAC3C,GAAG,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC;IACxC,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACrB,KAAK,GAAG,CAAC,CAAC;QACV,GAAG,GAAG,KAAK,CAAC;IACd,CAAC;SAAM,CAAC;QACN,SAAS,CACP,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC;YACrB,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC;YACrB,KAAK,IAAI,CAAC;YACV,GAAG,IAAI,KAAK;YACZ,KAAK,IAAI,KAAK;YACd,GAAG,IAAI,KAAK,EACd,iCAAiC,CAClC,CAAC;QACF,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,OAAO,CAAC,CAAC;QACrC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,GAAG,OAAO,CAAC,CAAC;IACvC,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC;IAC/D,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ;QACjC,CAAC,CAAC,IAAI,CAAC,UAAU;QACjB,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzC,OAAO;QACL,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,OAAO;QAChC,SAAS,EAAE,CAAC,CAAC,SAAS;QACtB,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,SAAS,EAAE,KAAK;QAChB,OAAO,EAAE,GAAG;QACZ,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,UAAU;QACV,QAAQ,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,IAAI,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,IAAI,IAAI,KAAK,IAAI,GAAG,EAAE;KAC7E,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@functional-systems/srcx",
|
|
3
|
-
"version": "0.1.0-dev.
|
|
3
|
+
"version": "0.1.0-dev.10",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "Version-aware code search on LambdaDB",
|
|
6
6
|
"type": "module",
|
|
@@ -28,11 +28,18 @@
|
|
|
28
28
|
"start": "node dist/cli.js",
|
|
29
29
|
"demo": "npm run build && node scripts/demo.mjs",
|
|
30
30
|
"prepack": "npm run build",
|
|
31
|
-
"format": "prettier --write src test scripts README.md VALIDATION.md CONTRIBUTING.md RELEASING.md CHANGELOG.md .github package.json tsconfig.json",
|
|
32
|
-
"format:check": "prettier --check src test scripts README.md VALIDATION.md CONTRIBUTING.md RELEASING.md CHANGELOG.md .github package.json tsconfig.json",
|
|
31
|
+
"format": "prettier --write src test scripts eval README.md VALIDATION.md CONTRIBUTING.md RELEASING.md CHANGELOG.md .github package.json tsconfig.json",
|
|
32
|
+
"format:check": "prettier --check src test scripts eval README.md VALIDATION.md CONTRIBUTING.md RELEASING.md CHANGELOG.md .github package.json tsconfig.json",
|
|
33
33
|
"test:live": "npm run build && node --env-file=.env.local scripts/live.mjs",
|
|
34
34
|
"check:version": "node scripts/check-release.mjs",
|
|
35
|
-
"test:package": "node scripts/test-package.mjs"
|
|
35
|
+
"test:package": "node scripts/test-package.mjs",
|
|
36
|
+
"test:live:branches": "npm run build && node --env-file=.env.local scripts/live-branches.mjs",
|
|
37
|
+
"eval:prepare": "npm run build && node scripts/retrieval-eval.mjs prepare",
|
|
38
|
+
"eval:run": "node scripts/retrieval-eval.mjs run",
|
|
39
|
+
"eval:cli:prepare": "npm run build && node scripts/cli-eval.mjs prepare",
|
|
40
|
+
"eval:cli:run": "node scripts/cli-eval.mjs run",
|
|
41
|
+
"eval:modes:prepare": "npm run build && node scripts/cli-eval.mjs prepare --suite eval/retrieval-modes-v1.json --root .srcx/mode-eval",
|
|
42
|
+
"eval:modes:run": "node scripts/cli-eval.mjs run --root .srcx/mode-eval"
|
|
36
43
|
},
|
|
37
44
|
"dependencies": {
|
|
38
45
|
"@functional-systems/lambdadb": "0.5.1",
|
|
@@ -54,5 +61,6 @@
|
|
|
54
61
|
"bugs": {
|
|
55
62
|
"url": "https://github.com/lambdadb/srcx/issues"
|
|
56
63
|
},
|
|
57
|
-
"license": "Apache-2.0"
|
|
64
|
+
"license": "Apache-2.0",
|
|
65
|
+
"gitHead": "a6583434ae3b934c5d091015dcb21268214bd651"
|
|
58
66
|
}
|
package/src/build.ts
CHANGED
|
@@ -27,8 +27,17 @@ export type Preset = {
|
|
|
27
27
|
schemaVersion: 1;
|
|
28
28
|
chunker: typeof CHUNKER;
|
|
29
29
|
mode: "syntax" | "window";
|
|
30
|
+
/** Internal evaluation override; the CLI keeps its original enrichment. */
|
|
31
|
+
enrichment?: "path-only-v1";
|
|
30
32
|
maxFileBytes: number;
|
|
31
|
-
embedding: {
|
|
33
|
+
embedding: {
|
|
34
|
+
model: string;
|
|
35
|
+
dimensions: number;
|
|
36
|
+
managed?: true;
|
|
37
|
+
provider?: "openai";
|
|
38
|
+
similarity?: "cosine";
|
|
39
|
+
sourceField?: "embeddingText";
|
|
40
|
+
} | null;
|
|
32
41
|
};
|
|
33
42
|
export const PRESET: Preset = {
|
|
34
43
|
schemaVersion: 1,
|
|
@@ -37,6 +46,78 @@ export const PRESET: Preset = {
|
|
|
37
46
|
maxFileBytes: 1024 * 1024,
|
|
38
47
|
embedding: null,
|
|
39
48
|
};
|
|
49
|
+
/** Opt-in managed preset; PRESET stays byte-for-byte compatible with lexical corpora. */
|
|
50
|
+
export const MANAGED_PRESET: Preset = {
|
|
51
|
+
...PRESET,
|
|
52
|
+
embedding: {
|
|
53
|
+
managed: true,
|
|
54
|
+
provider: "openai",
|
|
55
|
+
model: "text-embedding-3-small",
|
|
56
|
+
dimensions: 1536,
|
|
57
|
+
similarity: "cosine",
|
|
58
|
+
sourceField: "embeddingText",
|
|
59
|
+
},
|
|
60
|
+
};
|
|
61
|
+
export const MANAGED_LARGE_PRESET: Preset = {
|
|
62
|
+
...MANAGED_PRESET,
|
|
63
|
+
embedding: {
|
|
64
|
+
...MANAGED_PRESET.embedding!,
|
|
65
|
+
model: "text-embedding-3-large",
|
|
66
|
+
dimensions: 3072,
|
|
67
|
+
},
|
|
68
|
+
};
|
|
69
|
+
export function presetFor(embedding = "none"): Preset {
|
|
70
|
+
invariant(
|
|
71
|
+
["none", "text-embedding-3-small", "text-embedding-3-large"].includes(
|
|
72
|
+
embedding,
|
|
73
|
+
),
|
|
74
|
+
"Embedding must be none, text-embedding-3-small or text-embedding-3-large.",
|
|
75
|
+
);
|
|
76
|
+
if (embedding === "none") return PRESET;
|
|
77
|
+
return embedding === "text-embedding-3-large"
|
|
78
|
+
? MANAGED_LARGE_PRESET
|
|
79
|
+
: MANAGED_PRESET;
|
|
80
|
+
}
|
|
81
|
+
export function supportedPreset(preset: unknown): preset is Preset {
|
|
82
|
+
return (
|
|
83
|
+
preset !== undefined &&
|
|
84
|
+
[hash(PRESET), hash(MANAGED_PRESET), hash(MANAGED_LARGE_PRESET)].includes(
|
|
85
|
+
hash(preset),
|
|
86
|
+
)
|
|
87
|
+
);
|
|
88
|
+
}
|
|
89
|
+
export function indexConfigs(preset: Preset = PRESET) {
|
|
90
|
+
if (!preset.embedding?.managed) return INDEX_CONFIGS;
|
|
91
|
+
const { managed: _, ...embedding } = preset.embedding;
|
|
92
|
+
return {
|
|
93
|
+
...INDEX_CONFIGS,
|
|
94
|
+
embeddingText: { type: "text", analyzers: ["standard"] },
|
|
95
|
+
embedding: { type: "vector", managedEmbedding: true, embedding },
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
/** Validate server enrichment separately; all other payload fields remain exact. */
|
|
99
|
+
export function recordHash(doc: Doc, preset: Preset): string {
|
|
100
|
+
if (!preset.embedding?.managed) return hash(doc);
|
|
101
|
+
const { embedding, ...source } = doc;
|
|
102
|
+
if (doc.kind === "chunk" && doc.embeddingStatus === "managed") {
|
|
103
|
+
invariant(
|
|
104
|
+
typeof doc.embeddingText === "string" &&
|
|
105
|
+
doc.embeddingText === doc.searchText &&
|
|
106
|
+
doc.embeddingInputHash ===
|
|
107
|
+
hash([preset.embedding, doc.embeddingText]) &&
|
|
108
|
+
Array.isArray(embedding) &&
|
|
109
|
+
embedding.length === preset.embedding.dimensions &&
|
|
110
|
+
embedding.every((v) => typeof v === "number" && Number.isFinite(v)) &&
|
|
111
|
+
embedding.some((v) => v !== 0),
|
|
112
|
+
"Missing or invalid managed embedding.",
|
|
113
|
+
);
|
|
114
|
+
} else
|
|
115
|
+
invariant(
|
|
116
|
+
embedding === undefined && doc.embeddingText === undefined,
|
|
117
|
+
"Unexpected managed embedding on an ineligible record.",
|
|
118
|
+
);
|
|
119
|
+
return hash(source);
|
|
120
|
+
}
|
|
40
121
|
export type InventoryItem = {
|
|
41
122
|
path: string;
|
|
42
123
|
pathBase64: string;
|
|
@@ -52,6 +133,8 @@ export type InventoryItem = {
|
|
|
52
133
|
parseStatus?: string;
|
|
53
134
|
tokens?: number;
|
|
54
135
|
embedded?: number;
|
|
136
|
+
managed?: number;
|
|
137
|
+
managedTokens?: number;
|
|
55
138
|
skipped?: number;
|
|
56
139
|
};
|
|
57
140
|
export type Build = {
|
|
@@ -123,6 +206,7 @@ function exclude(path: string): string | undefined {
|
|
|
123
206
|
export async function materialize(args: {
|
|
124
207
|
identity: Identity;
|
|
125
208
|
ref: string;
|
|
209
|
+
resolvedCommit?: Awaited<ReturnType<typeof resolveCommit>>;
|
|
126
210
|
output: string;
|
|
127
211
|
previous?: Build;
|
|
128
212
|
preset?: Preset;
|
|
@@ -135,6 +219,8 @@ export async function materialize(args: {
|
|
|
135
219
|
invariant(
|
|
136
220
|
hash(preset.chunker) === hash(CHUNKER) &&
|
|
137
221
|
["syntax", "window"].includes(preset.mode) &&
|
|
222
|
+
(preset.enrichment === undefined ||
|
|
223
|
+
preset.enrichment === "path-only-v1") &&
|
|
138
224
|
preset.maxFileBytes === PRESET.maxFileBytes,
|
|
139
225
|
"Unsupported build preset.",
|
|
140
226
|
);
|
|
@@ -147,11 +233,13 @@ export async function materialize(args: {
|
|
|
147
233
|
}
|
|
148
234
|
invariant(
|
|
149
235
|
!preset.embedding ||
|
|
236
|
+
(preset.embedding.managed && supportedPreset(preset)) ||
|
|
150
237
|
(args.embedder?.model === preset.embedding.model &&
|
|
151
238
|
args.embedder.dimensions === preset.embedding.dimensions),
|
|
152
239
|
"Embedding provider must match the pinned preset.",
|
|
153
240
|
);
|
|
154
|
-
const commit =
|
|
241
|
+
const commit =
|
|
242
|
+
args.resolvedCommit ?? (await resolveCommit(args.identity.path, args.ref));
|
|
155
243
|
const directory = resolve(args.output);
|
|
156
244
|
await mkdir(directory, { recursive: false, mode: 0o700 });
|
|
157
245
|
const output = await open(join(directory, "records.jsonl"), "wx", 0o600);
|
|
@@ -233,7 +321,12 @@ export async function materialize(args: {
|
|
|
233
321
|
item.reason = "lfs-pointer";
|
|
234
322
|
continue;
|
|
235
323
|
}
|
|
236
|
-
const parsed = await chunk(
|
|
324
|
+
const parsed = await chunk(
|
|
325
|
+
source,
|
|
326
|
+
e.path,
|
|
327
|
+
preset.mode,
|
|
328
|
+
preset.enrichment,
|
|
329
|
+
);
|
|
237
330
|
const contentHash = hash(data);
|
|
238
331
|
const fileId = `f-${hash([e.pathBase64, contentHash])}`;
|
|
239
332
|
const base = { schemaVersion: 1, configHash };
|
|
@@ -285,31 +378,40 @@ export async function materialize(args: {
|
|
|
285
378
|
const eligible = !["imports", "structural"].includes(span.chunkKind);
|
|
286
379
|
if (preset.embedding && eligible) {
|
|
287
380
|
const inputHash = hash([preset.embedding, span.searchText]);
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
381
|
+
if (preset.embedding.managed) {
|
|
382
|
+
doc.embeddingText = span.searchText;
|
|
383
|
+
doc.embeddingInputHash = inputHash;
|
|
384
|
+
doc.embeddingStatus = "managed";
|
|
385
|
+
delete doc.embeddingSkipReason;
|
|
386
|
+
item.managed = (item.managed ?? 0) + 1;
|
|
387
|
+
item.managedTokens = (item.managedTokens ?? 0) + span.tokenCount;
|
|
388
|
+
} else {
|
|
389
|
+
const cachePath = args.cache
|
|
390
|
+
? join(args.cache, inputHash + ".json")
|
|
391
|
+
: undefined;
|
|
392
|
+
let vector = cachePath
|
|
393
|
+
? await optionalJson<number[]>(cachePath)
|
|
394
|
+
: undefined;
|
|
395
|
+
if (!vector) {
|
|
396
|
+
vector = await args.embedder!.embed(span.searchText);
|
|
397
|
+
invariant(
|
|
398
|
+
vector.length === preset.embedding.dimensions &&
|
|
399
|
+
vector.every(Number.isFinite),
|
|
400
|
+
"Invalid embedding output.",
|
|
401
|
+
);
|
|
402
|
+
if (cachePath) await atomic(cachePath, vector);
|
|
403
|
+
}
|
|
296
404
|
invariant(
|
|
297
405
|
vector.length === preset.embedding.dimensions &&
|
|
298
406
|
vector.every(Number.isFinite),
|
|
299
|
-
"Invalid
|
|
407
|
+
"Invalid cached vector.",
|
|
300
408
|
);
|
|
301
|
-
|
|
409
|
+
doc.embedding = vector;
|
|
410
|
+
doc.embeddingInputHash = inputHash;
|
|
411
|
+
doc.embeddingStatus = "embedded";
|
|
412
|
+
delete doc.embeddingSkipReason;
|
|
413
|
+
item.embedded!++;
|
|
302
414
|
}
|
|
303
|
-
invariant(
|
|
304
|
-
vector.length === preset.embedding.dimensions &&
|
|
305
|
-
vector.every(Number.isFinite),
|
|
306
|
-
"Invalid cached vector.",
|
|
307
|
-
);
|
|
308
|
-
doc.embedding = vector;
|
|
309
|
-
doc.embeddingInputHash = inputHash;
|
|
310
|
-
doc.embeddingStatus = "embedded";
|
|
311
|
-
delete doc.embeddingSkipReason;
|
|
312
|
-
item.embedded!++;
|
|
313
415
|
} else {
|
|
314
416
|
doc.embeddingSkipReason = preset.embedding
|
|
315
417
|
? span.chunkKind === "imports"
|
|
@@ -388,6 +490,15 @@ export async function materialize(args: {
|
|
|
388
490
|
excluded: items.filter((e) => e.status === "excluded").length,
|
|
389
491
|
chunks: items.reduce((n, e) => n + (e.chunkIds?.length ?? 0), 0),
|
|
390
492
|
tokens: items.reduce((n, e) => n + (e.tokens ?? 0), 0),
|
|
493
|
+
...(preset.embedding?.managed
|
|
494
|
+
? {
|
|
495
|
+
managed: items.reduce((n, e) => n + (e.managed ?? 0), 0),
|
|
496
|
+
managedTokens: items.reduce(
|
|
497
|
+
(n, e) => n + (e.managedTokens ?? 0),
|
|
498
|
+
0,
|
|
499
|
+
),
|
|
500
|
+
}
|
|
501
|
+
: {}),
|
|
391
502
|
embedded: items.reduce((n, e) => n + (e.embedded ?? 0), 0),
|
|
392
503
|
skipped: items.reduce((n, e) => n + (e.skipped ?? 0), 0),
|
|
393
504
|
},
|
|
@@ -439,6 +550,17 @@ export async function validateBuild(b: Build): Promise<void> {
|
|
|
439
550
|
doc.configHash === b.configHash && doc.schemaVersion === 1,
|
|
440
551
|
"Mixed configuration.",
|
|
441
552
|
);
|
|
553
|
+
if (b.preset.embedding?.managed) {
|
|
554
|
+
invariant(
|
|
555
|
+
doc.embedding === undefined,
|
|
556
|
+
"Managed artifacts must not contain vectors.",
|
|
557
|
+
);
|
|
558
|
+
if (doc.kind !== "chunk")
|
|
559
|
+
invariant(
|
|
560
|
+
doc.embeddingText === undefined,
|
|
561
|
+
"Only eligible chunks can contain embedding input.",
|
|
562
|
+
);
|
|
563
|
+
}
|
|
442
564
|
if (doc.kind === "file") {
|
|
443
565
|
finish();
|
|
444
566
|
activeFile = doc;
|
|
@@ -494,7 +616,27 @@ export async function validateBuild(b: Build): Promise<void> {
|
|
|
494
616
|
Number(doc.tokenCount) <= CHUNKER.maxTokens,
|
|
495
617
|
"Invalid chunk source/position/token binding.",
|
|
496
618
|
);
|
|
497
|
-
if (
|
|
619
|
+
if (b.preset.embedding?.managed) {
|
|
620
|
+
const eligible = !["imports", "structural"].includes(
|
|
621
|
+
String(doc.chunkKind),
|
|
622
|
+
);
|
|
623
|
+
invariant(
|
|
624
|
+
eligible
|
|
625
|
+
? doc.embeddingStatus === "managed" &&
|
|
626
|
+
doc.embeddingText === doc.searchText &&
|
|
627
|
+
doc.embeddingInputHash ===
|
|
628
|
+
hash([b.preset.embedding, doc.searchText]) &&
|
|
629
|
+
doc.embeddingSkipReason === undefined
|
|
630
|
+
: doc.embeddingStatus === "skipped" &&
|
|
631
|
+
doc.embeddingText === undefined &&
|
|
632
|
+
doc.embeddingInputHash === undefined &&
|
|
633
|
+
doc.embeddingSkipReason ===
|
|
634
|
+
(doc.chunkKind === "imports"
|
|
635
|
+
? "imports-only"
|
|
636
|
+
: "structural-only"),
|
|
637
|
+
"Invalid managed embedding input or eligibility.",
|
|
638
|
+
);
|
|
639
|
+
} else if (doc.embeddingStatus === "embedded")
|
|
498
640
|
invariant(
|
|
499
641
|
b.preset.embedding &&
|
|
500
642
|
Array.isArray(doc.embedding) &&
|
package/src/chunk.ts
CHANGED
|
@@ -75,7 +75,9 @@ export function detectLanguage(path: string): string {
|
|
|
75
75
|
)[ext ?? ""] ?? "text"
|
|
76
76
|
);
|
|
77
77
|
}
|
|
78
|
-
|
|
78
|
+
export type Enrichment = "path-scope-symbol-v1" | "path-only-v1";
|
|
79
|
+
function enrichment(path: string, u: Unit, policy: Enrichment): string {
|
|
80
|
+
if (policy === "path-only-v1") return path.slice(0, 240) + "\n";
|
|
79
81
|
return (
|
|
80
82
|
[path.slice(0, 240), u.scope?.slice(0, 160), u.symbol?.slice(0, 160)]
|
|
81
83
|
.filter(Boolean)
|
|
@@ -106,10 +108,11 @@ function splitUnit(
|
|
|
106
108
|
path: string,
|
|
107
109
|
u: Unit,
|
|
108
110
|
overlap: number,
|
|
111
|
+
policy: Enrichment,
|
|
109
112
|
): Unit[] {
|
|
110
113
|
const result: Unit[] = [];
|
|
111
114
|
let start = u.start;
|
|
112
|
-
const prefix = enrichment(path, u);
|
|
115
|
+
const prefix = enrichment(path, u, policy);
|
|
113
116
|
invariant(
|
|
114
117
|
tokens(prefix) < CHUNKER.maxTokens,
|
|
115
118
|
"Path/scope enrichment exceeds token limit.",
|
|
@@ -185,6 +188,7 @@ export async function chunk(
|
|
|
185
188
|
source: string,
|
|
186
189
|
path: string,
|
|
187
190
|
mode: "syntax" | "window" = "syntax",
|
|
191
|
+
policy: Enrichment = CHUNKER.enrichment,
|
|
188
192
|
): Promise<{ spans: Span[]; parseStatus: string; language: string }> {
|
|
189
193
|
const lang = detectLanguage(path);
|
|
190
194
|
if (!source.length)
|
|
@@ -224,7 +228,7 @@ export async function chunk(
|
|
|
224
228
|
if (
|
|
225
229
|
n.namedChildCount &&
|
|
226
230
|
(container ||
|
|
227
|
-
tokens(enrichment(path, u) + n.text) > CHUNKER.maxTokens)
|
|
231
|
+
tokens(enrichment(path, u, policy) + n.text) > CHUNKER.maxTokens)
|
|
228
232
|
) {
|
|
229
233
|
for (const child of n.namedChildren) if (child) visit(child);
|
|
230
234
|
} else if (n.endIndex > n.startIndex) units.push(u);
|
|
@@ -271,8 +275,9 @@ export async function chunk(
|
|
|
271
275
|
prev.kind !== "function" &&
|
|
272
276
|
u.kind === prev.kind &&
|
|
273
277
|
u.scope === prev.scope &&
|
|
274
|
-
tokens(
|
|
275
|
-
|
|
278
|
+
tokens(
|
|
279
|
+
enrichment(path, prev, policy) + source.slice(prev.start, u.end),
|
|
280
|
+
) <= CHUNKER.targetTokens
|
|
276
281
|
) {
|
|
277
282
|
prev.end = u.end;
|
|
278
283
|
delete prev.symbol;
|
|
@@ -280,7 +285,7 @@ export async function chunk(
|
|
|
280
285
|
} else merged.push({ ...u });
|
|
281
286
|
}
|
|
282
287
|
const bounded = merged.flatMap((u) =>
|
|
283
|
-
splitUnit(source, path, u, fallback ? CHUNKER.fallbackOverlap : 0),
|
|
288
|
+
splitUnit(source, path, u, fallback ? CHUNKER.fallbackOverlap : 0, policy),
|
|
284
289
|
);
|
|
285
290
|
const offsets = new Uint32Array(source.length + 1);
|
|
286
291
|
let byte = 0;
|
|
@@ -298,7 +303,7 @@ export async function chunk(
|
|
|
298
303
|
const startByte = offsets[u.start]!,
|
|
299
304
|
endByte = offsets[u.end]!;
|
|
300
305
|
const text = source.slice(u.start, u.end);
|
|
301
|
-
const searchText = enrichment(path, u) + text;
|
|
306
|
+
const searchText = enrichment(path, u, policy) + text;
|
|
302
307
|
const kind = text.trim() === "" ? "structural" : u.kind;
|
|
303
308
|
return {
|
|
304
309
|
startByte,
|