@functional-systems/srcx 0.1.0-dev.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/CONTRIBUTING.md +54 -0
- package/DESIGN.md +1057 -0
- package/LICENSE +202 -0
- package/README.md +209 -0
- package/RELEASING.md +137 -0
- package/VALIDATION.md +181 -0
- package/dist/build.d.ts +94 -0
- package/dist/build.js +433 -0
- package/dist/build.js.map +1 -0
- package/dist/chunk.d.ts +30 -0
- package/dist/chunk.js +296 -0
- package/dist/chunk.js.map +1 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +276 -0
- package/dist/cli.js.map +1 -0
- package/dist/common.d.ts +17 -0
- package/dist/common.js +74 -0
- package/dist/common.js.map +1 -0
- package/dist/git.d.ts +38 -0
- package/dist/git.js +224 -0
- package/dist/git.js.map +1 -0
- package/dist/publish.d.ts +60 -0
- package/dist/publish.js +341 -0
- package/dist/publish.js.map +1 -0
- package/dist/releases.d.ts +8 -0
- package/dist/releases.js +79 -0
- package/dist/releases.js.map +1 -0
- package/dist/remote.d.ts +53 -0
- package/dist/remote.js +136 -0
- package/dist/remote.js.map +1 -0
- package/dist/repository.d.ts +29 -0
- package/dist/repository.js +180 -0
- package/dist/repository.js.map +1 -0
- package/dist/search.d.ts +46 -0
- package/dist/search.js +181 -0
- package/dist/search.js.map +1 -0
- package/dist/settings.d.ts +10 -0
- package/dist/settings.js +35 -0
- package/dist/settings.js.map +1 -0
- package/package.json +58 -0
- package/src/build.ts +587 -0
- package/src/chunk.ts +328 -0
- package/src/cli.ts +366 -0
- package/src/common.ts +71 -0
- package/src/git.ts +286 -0
- package/src/publish.ts +487 -0
- package/src/releases.ts +108 -0
- package/src/remote.ts +230 -0
- package/src/repository.ts +306 -0
- package/src/search.ts +282 -0
- package/src/settings.ts +59 -0
package/dist/build.js
ADDED
|
@@ -0,0 +1,433 @@
|
|
|
1
|
+
import { mkdir, open, rm } from "node:fs/promises";
|
|
2
|
+
import { createReadStream } from "node:fs";
|
|
3
|
+
import { createInterface } from "node:readline";
|
|
4
|
+
import { join, resolve } from "node:path";
|
|
5
|
+
import { Blobs, inventory, resolveCommit } from "./git.js";
|
|
6
|
+
import { chunk, CHUNKER, tokens } from "./chunk.js";
|
|
7
|
+
import { atomic, canonical, hash, invariant, lineAt, json, utf8, optionalJson, } from "./common.js";
|
|
8
|
+
export const INDEX_CONFIGS = {
|
|
9
|
+
kind: { type: "keyword" },
|
|
10
|
+
path: { type: "keyword" },
|
|
11
|
+
language: { type: "keyword" },
|
|
12
|
+
fileId: { type: "keyword" },
|
|
13
|
+
symbol: { type: "keyword" },
|
|
14
|
+
searchText: { type: "text", analyzers: ["standard"] },
|
|
15
|
+
};
|
|
16
|
+
export const PRESET = {
|
|
17
|
+
schemaVersion: 1,
|
|
18
|
+
chunker: CHUNKER,
|
|
19
|
+
mode: "syntax",
|
|
20
|
+
maxFileBytes: 1024 * 1024,
|
|
21
|
+
embedding: null,
|
|
22
|
+
};
|
|
23
|
+
export async function* records(directory) {
|
|
24
|
+
const input = createReadStream(join(directory, "records.jsonl"), {
|
|
25
|
+
encoding: "utf8",
|
|
26
|
+
});
|
|
27
|
+
const lines = createInterface({ input, crlfDelay: Infinity });
|
|
28
|
+
try {
|
|
29
|
+
for await (const line of lines)
|
|
30
|
+
if (line)
|
|
31
|
+
yield JSON.parse(line);
|
|
32
|
+
}
|
|
33
|
+
finally {
|
|
34
|
+
lines.close();
|
|
35
|
+
input.destroy();
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
export async function loadBuild(directory) {
|
|
39
|
+
const b = await json(join(directory, "build.json"));
|
|
40
|
+
invariant(b.format === 1 && b.configHash === hash(b.preset), "Invalid build configuration.");
|
|
41
|
+
b.directory = resolve(directory);
|
|
42
|
+
return b;
|
|
43
|
+
}
|
|
44
|
+
function exclude(path) {
|
|
45
|
+
if (/(^|\/)(node_modules|vendor|vendored|dist|build|target|coverage|\.git|\.next)(\/|$)/.test(path))
|
|
46
|
+
return "dependency-or-build-output";
|
|
47
|
+
if (/\.(png|jpe?g|gif|ico|pdf|zip|gz|jar|class|woff2?|mp[34]|exe|dll|so|dylib|wasm)$/i.test(path))
|
|
48
|
+
return "binary-extension";
|
|
49
|
+
if (/\.(min\.(js|css)|map)$/.test(path))
|
|
50
|
+
return "generated-output";
|
|
51
|
+
return undefined;
|
|
52
|
+
}
|
|
53
|
+
export async function materialize(args) {
|
|
54
|
+
const preset = args.preset ?? PRESET, configHash = hash(preset), prev = args.previous;
|
|
55
|
+
invariant(hash(preset.chunker) === hash(CHUNKER) &&
|
|
56
|
+
["syntax", "window"].includes(preset.mode) &&
|
|
57
|
+
preset.maxFileBytes === PRESET.maxFileBytes, "Unsupported build preset.");
|
|
58
|
+
if (prev) {
|
|
59
|
+
invariant(prev.repoKey === args.identity.key && prev.configHash === configHash, "Previous build repository/config mismatch.");
|
|
60
|
+
await validateBuild(prev);
|
|
61
|
+
}
|
|
62
|
+
invariant(!preset.embedding ||
|
|
63
|
+
(args.embedder?.model === preset.embedding.model &&
|
|
64
|
+
args.embedder.dimensions === preset.embedding.dimensions), "Embedding provider must match the pinned preset.");
|
|
65
|
+
const commit = await resolveCommit(args.identity.path, args.ref);
|
|
66
|
+
const directory = resolve(args.output);
|
|
67
|
+
await mkdir(directory, { recursive: false, mode: 0o700 });
|
|
68
|
+
const output = await open(join(directory, "records.jsonl"), "wx", 0o600);
|
|
69
|
+
const recordHashes = {};
|
|
70
|
+
let writeError = false;
|
|
71
|
+
async function emit(doc) {
|
|
72
|
+
const data = canonical(doc);
|
|
73
|
+
invariant(Buffer.byteLength(data) <= 4_800_000, `Serialized document exceeds safe limit: ${doc.id}`);
|
|
74
|
+
invariant(!recordHashes[doc.id], "Duplicate document ID.");
|
|
75
|
+
recordHashes[doc.id] = hash(doc);
|
|
76
|
+
await output.write(data + "\n");
|
|
77
|
+
}
|
|
78
|
+
const items = [];
|
|
79
|
+
const blobs = new Blobs(args.identity.path);
|
|
80
|
+
const old = new Map(prev?.inventory.map((e) => [e.pathBase64, e]) ?? []);
|
|
81
|
+
const reuse = new Set();
|
|
82
|
+
try {
|
|
83
|
+
const entries = await inventory(args.identity.path, commit.oid);
|
|
84
|
+
for (const e of entries) {
|
|
85
|
+
const item = {
|
|
86
|
+
path: e.path,
|
|
87
|
+
pathBase64: e.pathBase64,
|
|
88
|
+
oid: e.oid,
|
|
89
|
+
mode: e.mode,
|
|
90
|
+
bytes: e.bytes,
|
|
91
|
+
status: "excluded",
|
|
92
|
+
};
|
|
93
|
+
items.push(item);
|
|
94
|
+
if (!e.validPath) {
|
|
95
|
+
item.reason = "invalid-utf8-path";
|
|
96
|
+
continue;
|
|
97
|
+
}
|
|
98
|
+
if (e.mode === "160000") {
|
|
99
|
+
item.reason = "submodule";
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
if (e.bytes > preset.maxFileBytes) {
|
|
103
|
+
item.reason = "oversized";
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
106
|
+
if (e.mode === "120000") {
|
|
107
|
+
item.reason = "symlink";
|
|
108
|
+
item.targetBase64 = (await blobs.read(e)).toString("base64");
|
|
109
|
+
continue;
|
|
110
|
+
}
|
|
111
|
+
const reason = exclude(e.path);
|
|
112
|
+
if (reason) {
|
|
113
|
+
item.reason = reason;
|
|
114
|
+
continue;
|
|
115
|
+
}
|
|
116
|
+
invariant(e.type === "blob", "Unsupported Git entry type.");
|
|
117
|
+
const former = old.get(e.pathBase64);
|
|
118
|
+
if (former?.status === "included" &&
|
|
119
|
+
former.oid === e.oid &&
|
|
120
|
+
former.mode === e.mode) {
|
|
121
|
+
Object.assign(item, former);
|
|
122
|
+
for (const id of [former.fileId, ...former.chunkIds])
|
|
123
|
+
reuse.add(id);
|
|
124
|
+
continue;
|
|
125
|
+
}
|
|
126
|
+
const data = await blobs.read(e);
|
|
127
|
+
if (data.includes(0)) {
|
|
128
|
+
item.reason = "binary-content";
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
const source = utf8(data);
|
|
132
|
+
if (source === undefined) {
|
|
133
|
+
item.reason = "invalid-utf8-content";
|
|
134
|
+
continue;
|
|
135
|
+
}
|
|
136
|
+
if (source.startsWith("version https://git-lfs.github.com/spec/v1\n") ||
|
|
137
|
+
source.startsWith("version https://git-lfs.github.com/spec/v1\r\n")) {
|
|
138
|
+
item.reason = "lfs-pointer";
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
const parsed = await chunk(source, e.path, preset.mode);
|
|
142
|
+
const contentHash = hash(data);
|
|
143
|
+
const fileId = `f-${hash([e.pathBase64, contentHash])}`;
|
|
144
|
+
const base = { schemaVersion: 1, configHash };
|
|
145
|
+
const file = {
|
|
146
|
+
...base,
|
|
147
|
+
id: fileId,
|
|
148
|
+
kind: "file",
|
|
149
|
+
path: e.path,
|
|
150
|
+
blobOid: e.oid,
|
|
151
|
+
contentHash,
|
|
152
|
+
byteLength: data.length,
|
|
153
|
+
sourceText: source,
|
|
154
|
+
language: parsed.language,
|
|
155
|
+
parseStatus: parsed.parseStatus,
|
|
156
|
+
chunkCount: parsed.spans.length,
|
|
157
|
+
};
|
|
158
|
+
if (Buffer.byteLength(canonical(file)) > 4_800_000) {
|
|
159
|
+
item.reason = "serialized-source-too-large";
|
|
160
|
+
continue;
|
|
161
|
+
}
|
|
162
|
+
Object.assign(item, {
|
|
163
|
+
status: "included",
|
|
164
|
+
contentHash,
|
|
165
|
+
fileId,
|
|
166
|
+
chunkIds: [],
|
|
167
|
+
parseStatus: parsed.parseStatus,
|
|
168
|
+
tokens: 0,
|
|
169
|
+
embedded: 0,
|
|
170
|
+
skipped: 0,
|
|
171
|
+
});
|
|
172
|
+
await emit(file);
|
|
173
|
+
for (const [ordinal, span] of parsed.spans.entries()) {
|
|
174
|
+
const id = `c-${hash([fileId, configHash, span.startByte, span.endByte, ordinal])}`;
|
|
175
|
+
const doc = {
|
|
176
|
+
...base,
|
|
177
|
+
id,
|
|
178
|
+
kind: "chunk",
|
|
179
|
+
fileId,
|
|
180
|
+
path: e.path,
|
|
181
|
+
contentHash,
|
|
182
|
+
language: parsed.language,
|
|
183
|
+
...span,
|
|
184
|
+
ordinal,
|
|
185
|
+
embeddingStatus: "skipped",
|
|
186
|
+
embeddingSkipReason: preset.embedding
|
|
187
|
+
? "structural-only"
|
|
188
|
+
: "embedding-disabled",
|
|
189
|
+
};
|
|
190
|
+
const eligible = !["imports", "structural"].includes(span.chunkKind);
|
|
191
|
+
if (preset.embedding && eligible) {
|
|
192
|
+
const inputHash = hash([preset.embedding, span.searchText]);
|
|
193
|
+
const cachePath = args.cache
|
|
194
|
+
? join(args.cache, inputHash + ".json")
|
|
195
|
+
: undefined;
|
|
196
|
+
let vector = cachePath
|
|
197
|
+
? await optionalJson(cachePath)
|
|
198
|
+
: undefined;
|
|
199
|
+
if (!vector) {
|
|
200
|
+
vector = await args.embedder.embed(span.searchText);
|
|
201
|
+
invariant(vector.length === preset.embedding.dimensions &&
|
|
202
|
+
vector.every(Number.isFinite), "Invalid embedding output.");
|
|
203
|
+
if (cachePath)
|
|
204
|
+
await atomic(cachePath, vector);
|
|
205
|
+
}
|
|
206
|
+
invariant(vector.length === preset.embedding.dimensions &&
|
|
207
|
+
vector.every(Number.isFinite), "Invalid cached vector.");
|
|
208
|
+
doc.embedding = vector;
|
|
209
|
+
doc.embeddingInputHash = inputHash;
|
|
210
|
+
doc.embeddingStatus = "embedded";
|
|
211
|
+
delete doc.embeddingSkipReason;
|
|
212
|
+
item.embedded++;
|
|
213
|
+
}
|
|
214
|
+
else {
|
|
215
|
+
doc.embeddingSkipReason = preset.embedding
|
|
216
|
+
? span.chunkKind === "imports"
|
|
217
|
+
? "imports-only"
|
|
218
|
+
: "structural-only"
|
|
219
|
+
: "embedding-disabled";
|
|
220
|
+
item.skipped++;
|
|
221
|
+
}
|
|
222
|
+
item.chunkIds.push(id);
|
|
223
|
+
item.tokens += span.tokenCount;
|
|
224
|
+
await emit(doc);
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
if (prev && reuse.size)
|
|
228
|
+
for await (const doc of records(prev.directory))
|
|
229
|
+
if (reuse.has(doc.id)) {
|
|
230
|
+
await emit(doc);
|
|
231
|
+
reuse.delete(doc.id);
|
|
232
|
+
}
|
|
233
|
+
invariant(reuse.size === 0, "Previous artifact lacks retained records.");
|
|
234
|
+
}
|
|
235
|
+
catch (e) {
|
|
236
|
+
writeError = true;
|
|
237
|
+
throw e;
|
|
238
|
+
}
|
|
239
|
+
finally {
|
|
240
|
+
blobs.close();
|
|
241
|
+
await output.close();
|
|
242
|
+
if (writeError)
|
|
243
|
+
await rm(directory, { recursive: true, force: true });
|
|
244
|
+
}
|
|
245
|
+
const changes = {
|
|
246
|
+
added: [],
|
|
247
|
+
modified: [],
|
|
248
|
+
deleted: [],
|
|
249
|
+
unchanged: [],
|
|
250
|
+
};
|
|
251
|
+
const now = new Map(items.map((e) => [e.pathBase64, e]));
|
|
252
|
+
for (const e of items) {
|
|
253
|
+
const o = old.get(e.pathBase64);
|
|
254
|
+
changes[!o
|
|
255
|
+
? "added"
|
|
256
|
+
: o.oid === e.oid && o.mode === e.mode && o.status === e.status
|
|
257
|
+
? "unchanged"
|
|
258
|
+
: "modified"].push(e.path);
|
|
259
|
+
}
|
|
260
|
+
for (const e of old.values())
|
|
261
|
+
if (!now.has(e.pathBase64))
|
|
262
|
+
changes.deleted.push(e.path);
|
|
263
|
+
const build = {
|
|
264
|
+
format: 1,
|
|
265
|
+
directory,
|
|
266
|
+
repoKey: args.identity.key,
|
|
267
|
+
configHash,
|
|
268
|
+
preset,
|
|
269
|
+
commitOid: commit.oid,
|
|
270
|
+
commitTime: commit.time,
|
|
271
|
+
requestedRef: args.ref,
|
|
272
|
+
branchRef: commit.branch,
|
|
273
|
+
buildId: hash([
|
|
274
|
+
args.identity.key,
|
|
275
|
+
commit.oid,
|
|
276
|
+
configHash,
|
|
277
|
+
"srcx-builder-v1",
|
|
278
|
+
]),
|
|
279
|
+
inventory: items,
|
|
280
|
+
recordHashes,
|
|
281
|
+
recordsHash: hash(Object.entries(recordHashes).sort(([a], [b]) => a.localeCompare(b))),
|
|
282
|
+
inventoryHash: hash(items),
|
|
283
|
+
changes,
|
|
284
|
+
obsoleteIds: Object.keys(prev?.recordHashes ?? {}).filter((id) => !recordHashes[id]),
|
|
285
|
+
counts: {
|
|
286
|
+
files: items.filter((e) => e.status === "included").length,
|
|
287
|
+
excluded: items.filter((e) => e.status === "excluded").length,
|
|
288
|
+
chunks: items.reduce((n, e) => n + (e.chunkIds?.length ?? 0), 0),
|
|
289
|
+
tokens: items.reduce((n, e) => n + (e.tokens ?? 0), 0),
|
|
290
|
+
embedded: items.reduce((n, e) => n + (e.embedded ?? 0), 0),
|
|
291
|
+
skipped: items.reduce((n, e) => n + (e.skipped ?? 0), 0),
|
|
292
|
+
},
|
|
293
|
+
};
|
|
294
|
+
await atomic(join(directory, "build.json"), build);
|
|
295
|
+
await validateBuild(build);
|
|
296
|
+
return build;
|
|
297
|
+
}
|
|
298
|
+
export async function validateBuild(b) {
|
|
299
|
+
invariant(typeof b.commitOid === "string" &&
|
|
300
|
+
/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(b.commitOid) &&
|
|
301
|
+
b.buildId ===
|
|
302
|
+
hash([b.repoKey, b.commitOid, b.configHash, "srcx-builder-v1"]), "Build commit identity mismatch; rebuild from the intended Git commit.");
|
|
303
|
+
invariant(hash(b.preset) === b.configHash && hash(b.inventory) === b.inventoryHash, "Build metadata hash mismatch.");
|
|
304
|
+
const seen = new Set();
|
|
305
|
+
let activeFile;
|
|
306
|
+
let end = 0, count = 0;
|
|
307
|
+
const expectedFile = new Map(b.inventory
|
|
308
|
+
.filter((e) => e.status === "included")
|
|
309
|
+
.map((e) => [e.fileId, e]));
|
|
310
|
+
const completedFiles = new Set();
|
|
311
|
+
const finish = () => {
|
|
312
|
+
if (activeFile) {
|
|
313
|
+
const entry = expectedFile.get(activeFile.id);
|
|
314
|
+
invariant(count === entry.chunkIds.length &&
|
|
315
|
+
end === Buffer.byteLength(activeFile.sourceText), "Incomplete source coverage.");
|
|
316
|
+
completedFiles.add(activeFile.id);
|
|
317
|
+
}
|
|
318
|
+
};
|
|
319
|
+
for await (const doc of records(b.directory)) {
|
|
320
|
+
invariant(!seen.has(doc.id) && b.recordHashes[doc.id] === hash(doc), "Corrupted or unexpected artifact record.");
|
|
321
|
+
seen.add(doc.id);
|
|
322
|
+
invariant(doc.configHash === b.configHash && doc.schemaVersion === 1, "Mixed configuration.");
|
|
323
|
+
if (doc.kind === "file") {
|
|
324
|
+
finish();
|
|
325
|
+
activeFile = doc;
|
|
326
|
+
end = 0;
|
|
327
|
+
count = 0;
|
|
328
|
+
const entry = expectedFile.get(doc.id);
|
|
329
|
+
const raw = Buffer.from(doc.sourceText);
|
|
330
|
+
invariant(entry &&
|
|
331
|
+
hash(raw) === doc.contentHash &&
|
|
332
|
+
entry.contentHash === doc.contentHash &&
|
|
333
|
+
entry.path === doc.path &&
|
|
334
|
+
entry.oid === doc.blobOid &&
|
|
335
|
+
entry.bytes === raw.length &&
|
|
336
|
+
doc.byteLength === raw.length &&
|
|
337
|
+
doc.chunkCount === entry.chunkIds?.length &&
|
|
338
|
+
doc.id === `f-${hash([entry.pathBase64, doc.contentHash])}`, "Invalid file content or inventory binding.");
|
|
339
|
+
}
|
|
340
|
+
else {
|
|
341
|
+
invariant(doc.kind === "chunk" && activeFile && doc.fileId === activeFile.id, "Orphan chunk.");
|
|
342
|
+
const a = doc.startByte, z = doc.endByte;
|
|
343
|
+
invariant(Number.isInteger(a) &&
|
|
344
|
+
Number.isInteger(z) &&
|
|
345
|
+
a >= 0 &&
|
|
346
|
+
a <= end &&
|
|
347
|
+
z > a &&
|
|
348
|
+
z <= Buffer.byteLength(activeFile.sourceText), "Invalid chunk range.");
|
|
349
|
+
invariant(expectedFile.get(activeFile.id).chunkIds.includes(doc.id), "Unexpected chunk membership.");
|
|
350
|
+
const raw = Buffer.from(activeFile.sourceText);
|
|
351
|
+
const slice = raw.subarray(a, z);
|
|
352
|
+
invariant(utf8(slice) !== undefined &&
|
|
353
|
+
doc.path === activeFile.path &&
|
|
354
|
+
doc.contentHash === activeFile.contentHash &&
|
|
355
|
+
doc.ordinal === count &&
|
|
356
|
+
doc.id === `c-${hash([activeFile.id, b.configHash, a, z, count])}` &&
|
|
357
|
+
doc.startLine === lineAt(raw, a) &&
|
|
358
|
+
doc.endLine === lineAt(raw, z - 1) &&
|
|
359
|
+
typeof doc.searchText === "string" &&
|
|
360
|
+
doc.searchText.endsWith(slice.toString("utf8")) &&
|
|
361
|
+
doc.tokenCount === tokens(doc.searchText) &&
|
|
362
|
+
Number(doc.tokenCount) <= CHUNKER.maxTokens, "Invalid chunk source/position/token binding.");
|
|
363
|
+
if (doc.embeddingStatus === "embedded")
|
|
364
|
+
invariant(b.preset.embedding &&
|
|
365
|
+
Array.isArray(doc.embedding) &&
|
|
366
|
+
doc.embedding.length === b.preset.embedding.dimensions &&
|
|
367
|
+
doc.embedding.every(Number.isFinite), "Invalid embedded chunk.");
|
|
368
|
+
else
|
|
369
|
+
invariant(doc.embeddingStatus === "skipped" &&
|
|
370
|
+
typeof doc.embeddingSkipReason === "string" &&
|
|
371
|
+
doc.embedding === undefined, "Unaccounted embedding state.");
|
|
372
|
+
end = Math.max(end, z);
|
|
373
|
+
count++;
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
finish();
|
|
377
|
+
invariant(completedFiles.size === expectedFile.size, "Missing inventory file records.");
|
|
378
|
+
invariant(seen.size === Object.keys(b.recordHashes).length, "Missing artifact records.");
|
|
379
|
+
invariant(hash(Object.entries(b.recordHashes).sort(([a], [b]) => a.localeCompare(b))) === b.recordsHash, "Record inventory hash mismatch.");
|
|
380
|
+
}
|
|
381
|
+
export function corpusMetadata(b, repoId, indexId, attemptId) {
|
|
382
|
+
const parts = [];
|
|
383
|
+
let entries = [];
|
|
384
|
+
let bytes = 0;
|
|
385
|
+
const flush = () => {
|
|
386
|
+
if (entries.length) {
|
|
387
|
+
const n = parts.length;
|
|
388
|
+
parts.push({
|
|
389
|
+
id: `inventory-${n}`,
|
|
390
|
+
kind: "manifest",
|
|
391
|
+
role: "inventory",
|
|
392
|
+
schemaVersion: 1,
|
|
393
|
+
configHash: b.configHash,
|
|
394
|
+
partOrdinal: n,
|
|
395
|
+
partHash: hash(entries),
|
|
396
|
+
entries,
|
|
397
|
+
});
|
|
398
|
+
entries = [];
|
|
399
|
+
bytes = 0;
|
|
400
|
+
}
|
|
401
|
+
};
|
|
402
|
+
for (const entry of b.inventory) {
|
|
403
|
+
const size = Buffer.byteLength(canonical(entry));
|
|
404
|
+
invariant(size < 4_000_000, "Single inventory entry too large.");
|
|
405
|
+
if (bytes + size > 1_000_000)
|
|
406
|
+
flush();
|
|
407
|
+
entries.push(entry);
|
|
408
|
+
bytes += size;
|
|
409
|
+
}
|
|
410
|
+
flush();
|
|
411
|
+
const root = {
|
|
412
|
+
id: "__manifest__",
|
|
413
|
+
kind: "manifest",
|
|
414
|
+
role: "corpus",
|
|
415
|
+
schemaVersion: 1,
|
|
416
|
+
configHash: b.configHash,
|
|
417
|
+
repoId,
|
|
418
|
+
indexId,
|
|
419
|
+
commitOid: b.commitOid,
|
|
420
|
+
commitTime: b.commitTime,
|
|
421
|
+
// Normalize the remote manifest so two ref spellings of one commit reuse it.
|
|
422
|
+
requestedRef: b.commitOid,
|
|
423
|
+
config: b.preset,
|
|
424
|
+
inventoryHash: b.inventoryHash,
|
|
425
|
+
recordsHash: b.recordsHash,
|
|
426
|
+
counts: b.counts,
|
|
427
|
+
buildId: b.buildId,
|
|
428
|
+
attemptId,
|
|
429
|
+
inventoryPartIds: parts.map((p) => p.id),
|
|
430
|
+
};
|
|
431
|
+
return [...parts, root];
|
|
432
|
+
}
|
|
433
|
+
//# sourceMappingURL=build.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"build.js","sourceRoot":"","sources":["../src/build.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,aAAa,EAAiB,MAAM,UAAU,CAAC;AAC1E,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AACpD,OAAO,EACL,MAAM,EACN,SAAS,EACT,IAAI,EACJ,SAAS,EACT,MAAM,EACN,IAAI,EACJ,IAAI,EAEJ,YAAY,GACb,MAAM,aAAa,CAAC;AACrB,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;IACzB,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;IACzB,QAAQ,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;IAC7B,MAAM,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;IAC3B,MAAM,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;IAC3B,UAAU,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,UAAU,CAAC,EAAE;CAC7C,CAAC;AAQX,MAAM,CAAC,MAAM,MAAM,GAAW;IAC5B,aAAa,EAAE,CAAC;IAChB,OAAO,EAAE,OAAO;IAChB,IAAI,EAAE,QAAQ;IACd,YAAY,EAAE,IAAI,GAAG,IAAI;IACzB,SAAS,EAAE,IAAI;CAChB,CAAC;AA+CF,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,OAAO,CAAC,SAAiB;IAC9C,MAAM,KAAK,GAAG,gBAAgB,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE;QAC/D,QAAQ,EAAE,MAAM;KACjB,CAAC,CAAC;IACH,MAAM,KAAK,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC9D,IAAI,CAAC;QACH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,KAAK;YAAE,IAAI,IAAI;gBAAE,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAQ,CAAC;IAC1E,CAAC;YAAS,CAAC;QACT,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,KAAK,CAAC,OAAO,EAAE,CAAC;IAClB,CAAC;AACH,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,SAAiB;IAC/C,MAAM,CAAC,GAAG,MAAM,IAAI,CAAQ,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC,CAAC;IAC3D,SAAS,CACP,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC,UAAU,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,EACjD,8BAA8B,CAC/B,CAAC;IACF,CAAC,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IACjC,OAAO,CAAC,CAAC;AACX,CAAC;AACD,SAAS,OAAO,CAAC,IAAY;IAC3B,IACE,oFAAoF,CAAC,IAAI,CACvF,IAAI,CACL;QAED,OAAO,4BAA4B,CAAC;IACtC,IACE,kFAAkF,CAAC,IAAI,CACrF,IAAI,CACL;QAED,OAAO,kBAAkB,CAAC;IAC5B,IAAI,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,kBAAkB,CAAC;IACnE,OAAO,SAAS,CAAC;AACnB,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAQjC;IACC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,MAAM,EAClC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,EACzB,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC;IACvB,SAAS,CACP,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC;QACpC,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC;QAC1C,MAAM,CAAC,YAAY,KAAK,MAAM,CAAC,YAAY,EAC7C,2BAA2B,CAC5B,CAAC;IACF,IAAI,IAAI,EAAE,CAAC;QACT,SAAS,CACP,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,QAAQ,CAAC,GAAG,IAAI,IAAI,CAAC,UAAU,KAAK,UAAU,EACpE,4CAA4C,CAC7C,CAAC;QACF,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IACD,SAAS,CACP,CAAC,MAAM,CAAC,SAAS;QACf,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,KAAK,MAAM,CAAC,SAAS,CAAC,KAAK;YAC9C,IAAI,CAAC,QAAQ,CAAC,UAAU,KAAK,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,EAC7D,kDAAkD,CACnD,CAAC;IACF,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IACjE,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvC,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAC1D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;IACzE,MAAM,YAAY,GAA2B,EAAE,CAAC;IAChD,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,KAAK,UAAU,IAAI,CAAC,GAAQ;QAC1B,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;QAC5B,SAAS,CACP,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,SAAS,EACpC,2CAA2C,GAAG,CAAC,EAAE,EAAE,CACpD,CAAC;QACF,SAAS,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,wBAAwB,CAAC,CAAC;QAC3D,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;QACjC,MAAM,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;IAClC,CAAC;IACD,MAAM,KAAK,GAAoB,EAAE,CAAC;IAClC,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC5C,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACzE,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;QAChE,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,IAAI,GAAkB;gBAC1B,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,UAAU,EAAE,CAAC,CAAC,UAAU;gBACxB,GAAG,EAAE,CAAC,CAAC,GAAG;gBACV,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,MAAM,EAAE,UAAU;aACnB,CAAC;YACF,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,IAAI,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;gBACjB,IAAI,CAAC,MAAM,GAAG,mBAAmB,CAAC;gBAClC,SAAS;YACX,CAAC;YACD,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACxB,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC;gBAC1B,SAAS;YACX,CAAC;YACD,IAAI,CAAC,CAAC,KAAK,GAAG,MAAM,CAAC,YAAY,EAAE,CAAC;gBAClC,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC;gBAC1B,SAAS;YACX,CAAC;YACD,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACxB,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;gBACxB,IAAI,CAAC,YAAY,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;gBAC7D,SAAS;YACX,CAAC;YACD,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC/B,IAAI,MAAM,EAAE,CAAC;gBACX,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;gBACrB,SAAS;YACX,CAAC;YACD,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,EAAE,6BAA6B,CAAC,CAAC;YAC5D,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;YACrC,IACE,MAAM,EAAE,MAAM,KAAK,UAAU;gBAC7B,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG;gBACpB,MAAM,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,EACtB,CAAC;gBACD,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;gBAC5B,KAAK,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAO,EAAE,GAAG,MAAM,CAAC,QAAS,CAAC;oBAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACtE,SAAS;YACX,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjC,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrB,IAAI,CAAC,MAAM,GAAG,gBAAgB,CAAC;gBAC/B,SAAS;YACX,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1B,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gBACzB,IAAI,CAAC,MAAM,GAAG,sBAAsB,CAAC;gBACrC,SAAS;YACX,CAAC;YACD,IACE,MAAM,CAAC,UAAU,CAAC,8CAA8C,CAAC;gBACjE,MAAM,CAAC,UAAU,CAAC,gDAAgD,CAAC,EACnE,CAAC;gBACD,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC;gBAC5B,SAAS;YACX,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YACxD,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/B,MAAM,MAAM,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC;YACxD,MAAM,IAAI,GAAG,EAAE,aAAa,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC;YAC9C,MAAM,IAAI,GAAQ;gBAChB,GAAG,IAAI;gBACP,EAAE,EAAE,MAAM;gBACV,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,OAAO,EAAE,CAAC,CAAC,GAAG;gBACd,WAAW;gBACX,UAAU,EAAE,IAAI,CAAC,MAAM;gBACvB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM;aAChC,CAAC;YACF,IAAI,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,SAAS,EAAE,CAAC;gBACnD,IAAI,CAAC,MAAM,GAAG,6BAA6B,CAAC;gBAC5C,SAAS;YACX,CAAC;YACD,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE;gBAClB,MAAM,EAAE,UAAU;gBAClB,WAAW;gBACX,MAAM;gBACN,QAAQ,EAAE,EAAE;gBACZ,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,MAAM,EAAE,CAAC;gBACT,QAAQ,EAAE,CAAC;gBACX,OAAO,EAAE,CAAC;aACX,CAAC,CAAC;YACH,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,KAAK,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;gBACrD,MAAM,EAAE,GAAG,KAAK,IAAI,CAAC,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,EAAE,CAAC;gBACpF,MAAM,GAAG,GAAQ;oBACf,GAAG,IAAI;oBACP,EAAE;oBACF,IAAI,EAAE,OAAO;oBACb,MAAM;oBACN,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,WAAW;oBACX,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,GAAG,IAAI;oBACP,OAAO;oBACP,eAAe,EAAE,SAAS;oBAC1B,mBAAmB,EAAE,MAAM,CAAC,SAAS;wBACnC,CAAC,CAAC,iBAAiB;wBACnB,CAAC,CAAC,oBAAoB;iBACzB,CAAC;gBACF,MAAM,QAAQ,GAAG,CAAC,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACrE,IAAI,MAAM,CAAC,SAAS,IAAI,QAAQ,EAAE,CAAC;oBACjC,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;oBAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK;wBAC1B,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC;wBACvC,CAAC,CAAC,SAAS,CAAC;oBACd,IAAI,MAAM,GAAG,SAAS;wBACpB,CAAC,CAAC,MAAM,YAAY,CAAW,SAAS,CAAC;wBACzC,CAAC,CAAC,SAAS,CAAC;oBACd,IAAI,CAAC,MAAM,EAAE,CAAC;wBACZ,MAAM,GAAG,MAAM,IAAI,CAAC,QAAS,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;wBACrD,SAAS,CACP,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,SAAS,CAAC,UAAU;4BAC3C,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,EAC/B,2BAA2B,CAC5B,CAAC;wBACF,IAAI,SAAS;4BAAE,MAAM,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;oBACjD,CAAC;oBACD,SAAS,CACP,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,SAAS,CAAC,UAAU;wBAC3C,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,EAC/B,wBAAwB,CACzB,CAAC;oBACF,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC;oBACvB,GAAG,CAAC,kBAAkB,GAAG,SAAS,CAAC;oBACnC,GAAG,CAAC,eAAe,GAAG,UAAU,CAAC;oBACjC,OAAO,GAAG,CAAC,mBAAmB,CAAC;oBAC/B,IAAI,CAAC,QAAS,EAAE,CAAC;gBACnB,CAAC;qBAAM,CAAC;oBACN,GAAG,CAAC,mBAAmB,GAAG,MAAM,CAAC,SAAS;wBACxC,CAAC,CAAC,IAAI,CAAC,SAAS,KAAK,SAAS;4BAC5B,CAAC,CAAC,cAAc;4BAChB,CAAC,CAAC,iBAAiB;wBACrB,CAAC,CAAC,oBAAoB,CAAC;oBACzB,IAAI,CAAC,OAAQ,EAAE,CAAC;gBAClB,CAAC;gBACD,IAAI,CAAC,QAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACxB,IAAI,CAAC,MAAO,IAAI,IAAI,CAAC,UAAU,CAAC;gBAChC,MAAM,IAAI,CAAC,GAAG,CAAC,CAAC;YAClB,CAAC;QACH,CAAC;QACD,IAAI,IAAI,IAAI,KAAK,CAAC,IAAI;YACpB,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC;gBAC7C,IAAI,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;oBACtB,MAAM,IAAI,CAAC,GAAG,CAAC,CAAC;oBAChB,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACvB,CAAC;QACL,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,EAAE,2CAA2C,CAAC,CAAC;IAC3E,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,UAAU,GAAG,IAAI,CAAC;QAClB,MAAM,CAAC,CAAC;IACV,CAAC;YAAS,CAAC;QACT,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACrB,IAAI,UAAU;YAAE,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACxE,CAAC;IACD,MAAM,OAAO,GAAqB;QAChC,KAAK,EAAE,EAAE;QACT,QAAQ,EAAE,EAAE;QACZ,OAAO,EAAE,EAAE;QACX,SAAS,EAAE,EAAE;KACd,CAAC;IACF,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACzD,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QAChC,OAAO,CACL,CAAC,CAAC;YACA,CAAC,CAAC,OAAO;YACT,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;gBAC7D,CAAC,CAAC,WAAW;gBACb,CAAC,CAAC,UAAU,CACjB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACjB,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,EAAE;QAC1B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC;YAAE,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAU;QACnB,MAAM,EAAE,CAAC;QACT,SAAS;QACT,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG;QAC1B,UAAU;QACV,MAAM;QACN,SAAS,EAAE,MAAM,CAAC,GAAG;QACrB,UAAU,EAAE,MAAM,CAAC,IAAI;QACvB,YAAY,EAAE,IAAI,CAAC,GAAG;QACtB,SAAS,EAAE,MAAM,CAAC,MAAM;QACxB,OAAO,EAAE,IAAI,CAAC;YACZ,IAAI,CAAC,QAAQ,CAAC,GAAG;YACjB,MAAM,CAAC,GAAG;YACV,UAAU;YACV,iBAAiB;SAClB,CAAC;QACF,SAAS,EAAE,KAAK;QAChB,YAAY;QACZ,WAAW,EAAE,IAAI,CACf,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CACpE;QACD,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC;QAC1B,OAAO;QACP,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,IAAI,EAAE,CAAC,CAAC,MAAM,CACvD,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,EAAE,CAAC,CAC1B;QACD,MAAM,EAAE;YACN,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,MAAM;YAC1D,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,MAAM;YAC7D,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;YAChE,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;YACtD,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;YAC1D,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;SACzD;KACF,CAAC;IACF,MAAM,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,EAAE,KAAK,CAAC,CAAC;IACnD,MAAM,aAAa,CAAC,KAAK,CAAC,CAAC;IAC3B,OAAO,KAAK,CAAC;AACf,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,CAAQ;IAC1C,SAAS,CACP,OAAO,CAAC,CAAC,SAAS,KAAK,QAAQ;QAC7B,iCAAiC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;QACnD,CAAC,CAAC,OAAO;YACP,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,UAAU,EAAE,iBAAiB,CAAC,CAAC,EACnE,uEAAuE,CACxE,CAAC;IACF,SAAS,CACP,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,UAAU,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,aAAa,EACxE,+BAA+B,CAChC,CAAC;IACF,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,IAAI,UAA2B,CAAC;IAChC,IAAI,GAAG,GAAG,CAAC,EACT,KAAK,GAAG,CAAC,CAAC;IACZ,MAAM,YAAY,GAAG,IAAI,GAAG,CAC1B,CAAC,CAAC,SAAS;SACR,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC;SACtC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAO,EAAE,CAAC,CAAC,CAAC,CAC9B,CAAC;IACF,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IACzC,MAAM,MAAM,GAAG,GAAG,EAAE;QAClB,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAE,CAAC;YAC/C,SAAS,CACP,KAAK,KAAK,KAAK,CAAC,QAAS,CAAC,MAAM;gBAC9B,GAAG,KAAK,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,UAAoB,CAAC,EAC5D,6BAA6B,CAC9B,CAAC;YACF,cAAc,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACpC,CAAC;IACH,CAAC,CAAC;IACF,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC;QAC7C,SAAS,CACP,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,IAAI,CAAC,GAAG,CAAC,EACzD,0CAA0C,CAC3C,CAAC;QACF,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACjB,SAAS,CACP,GAAG,CAAC,UAAU,KAAK,CAAC,CAAC,UAAU,IAAI,GAAG,CAAC,aAAa,KAAK,CAAC,EAC1D,sBAAsB,CACvB,CAAC;QACF,IAAI,GAAG,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACxB,MAAM,EAAE,CAAC;YACT,UAAU,GAAG,GAAG,CAAC;YACjB,GAAG,GAAG,CAAC,CAAC;YACR,KAAK,GAAG,CAAC,CAAC;YACV,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACvC,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,UAAoB,CAAC,CAAC;YAClD,SAAS,CACP,KAAK;gBACH,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,WAAW;gBAC7B,KAAK,CAAC,WAAW,KAAK,GAAG,CAAC,WAAW;gBACrC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI;gBACvB,KAAK,CAAC,GAAG,KAAK,GAAG,CAAC,OAAO;gBACzB,KAAK,CAAC,KAAK,KAAK,GAAG,CAAC,MAAM;gBAC1B,GAAG,CAAC,UAAU,KAAK,GAAG,CAAC,MAAM;gBAC7B,GAAG,CAAC,UAAU,KAAK,KAAK,CAAC,QAAQ,EAAE,MAAM;gBACzC,GAAG,CAAC,EAAE,KAAK,KAAK,IAAI,CAAC,CAAC,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC,EAAE,EAC7D,4CAA4C,CAC7C,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,SAAS,CACP,GAAG,CAAC,IAAI,KAAK,OAAO,IAAI,UAAU,IAAI,GAAG,CAAC,MAAM,KAAK,UAAU,CAAC,EAAE,EAClE,eAAe,CAChB,CAAC;YACF,MAAM,CAAC,GAAG,GAAG,CAAC,SAAmB,EAC/B,CAAC,GAAG,GAAG,CAAC,OAAiB,CAAC;YAC5B,SAAS,CACP,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;gBACjB,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;gBACnB,CAAC,IAAI,CAAC;gBACN,CAAC,IAAI,GAAG;gBACR,CAAC,GAAG,CAAC;gBACL,CAAC,IAAI,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,UAAoB,CAAC,EACzD,sBAAsB,CACvB,CAAC;YACF,SAAS,CACP,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAE,CAAC,QAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,EAC3D,8BAA8B,CAC/B,CAAC;YACF,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,UAAoB,CAAC,CAAC;YACzD,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACjC,SAAS,CACP,IAAI,CAAC,KAAK,CAAC,KAAK,SAAS;gBACvB,GAAG,CAAC,IAAI,KAAK,UAAU,CAAC,IAAI;gBAC5B,GAAG,CAAC,WAAW,KAAK,UAAU,CAAC,WAAW;gBAC1C,GAAG,CAAC,OAAO,KAAK,KAAK;gBACrB,GAAG,CAAC,EAAE,KAAK,KAAK,IAAI,CAAC,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE;gBAClE,GAAG,CAAC,SAAS,KAAK,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;gBAChC,GAAG,CAAC,OAAO,KAAK,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC;gBAClC,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ;gBAClC,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;gBAC/C,GAAG,CAAC,UAAU,KAAK,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC;gBACzC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC,SAAS,EAC7C,8CAA8C,CAC/C,CAAC;YACF,IAAI,GAAG,CAAC,eAAe,KAAK,UAAU;gBACpC,SAAS,CACP,CAAC,CAAC,MAAM,CAAC,SAAS;oBAChB,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC;oBAC5B,GAAG,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU;oBACtD,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,EACtC,yBAAyB,CAC1B,CAAC;;gBAEF,SAAS,CACP,GAAG,CAAC,eAAe,KAAK,SAAS;oBAC/B,OAAO,GAAG,CAAC,mBAAmB,KAAK,QAAQ;oBAC3C,GAAG,CAAC,SAAS,KAAK,SAAS,EAC7B,8BAA8B,CAC/B,CAAC;YACJ,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YACvB,KAAK,EAAE,CAAC;QACV,CAAC;IACH,CAAC;IACD,MAAM,EAAE,CAAC;IACT,SAAS,CACP,cAAc,CAAC,IAAI,KAAK,YAAY,CAAC,IAAI,EACzC,iCAAiC,CAClC,CAAC;IACF,SAAS,CACP,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,MAAM,EAChD,2BAA2B,CAC5B,CAAC;IACF,SAAS,CACP,IAAI,CACF,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CACtE,KAAK,CAAC,CAAC,WAAW,EACnB,iCAAiC,CAClC,CAAC;AACJ,CAAC;AACD,MAAM,UAAU,cAAc,CAC5B,CAAQ,EACR,MAAc,EACd,OAAe,EACf,SAAiB;IAEjB,MAAM,KAAK,GAAU,EAAE,CAAC;IACxB,IAAI,OAAO,GAAoB,EAAE,CAAC;IAClC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,MAAM,KAAK,GAAG,GAAG,EAAE;QACjB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;YACvB,KAAK,CAAC,IAAI,CAAC;gBACT,EAAE,EAAE,aAAa,CAAC,EAAE;gBACpB,IAAI,EAAE,UAAU;gBAChB,IAAI,EAAE,WAAW;gBACjB,aAAa,EAAE,CAAC;gBAChB,UAAU,EAAE,CAAC,CAAC,UAAU;gBACxB,WAAW,EAAE,CAAC;gBACd,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC;gBACvB,OAAO;aACR,CAAC,CAAC;YACH,OAAO,GAAG,EAAE,CAAC;YACb,KAAK,GAAG,CAAC,CAAC;QACZ,CAAC;IACH,CAAC,CAAC;IACF,KAAK,MAAM,KAAK,IAAI,CAAC,CAAC,SAAS,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;QACjD,SAAS,CAAC,IAAI,GAAG,SAAS,EAAE,mCAAmC,CAAC,CAAC;QACjE,IAAI,KAAK,GAAG,IAAI,GAAG,SAAS;YAAE,KAAK,EAAE,CAAC;QACtC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpB,KAAK,IAAI,IAAI,CAAC;IAChB,CAAC;IACD,KAAK,EAAE,CAAC;IACR,MAAM,IAAI,GAAQ;QAChB,EAAE,EAAE,cAAc;QAClB,IAAI,EAAE,UAAU;QAChB,IAAI,EAAE,QAAQ;QACd,aAAa,EAAE,CAAC;QAChB,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,MAAM;QACN,OAAO;QACP,SAAS,EAAE,CAAC,CAAC,SAAS;QACtB,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,6EAA6E;QAC7E,YAAY,EAAE,CAAC,CAAC,SAAS;QACzB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,aAAa,EAAE,CAAC,CAAC,aAAa;QAC9B,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,SAAS;QACT,gBAAgB,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACzC,CAAC;IACF,OAAO,CAAC,GAAG,KAAK,EAAE,IAAI,CAAC,CAAC;AAC1B,CAAC"}
|
package/dist/chunk.d.ts
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
export declare function tokens(text: string): number;
|
|
2
|
+
export declare const CHUNKER: {
|
|
3
|
+
readonly version: 1;
|
|
4
|
+
readonly parser: "web-tree-sitter@0.25.10";
|
|
5
|
+
readonly grammars: "tree-sitter-wasms@0.1.13";
|
|
6
|
+
readonly tokenizer: "js-tiktoken@1.0.21/cl100k_base";
|
|
7
|
+
readonly targetTokens: 800;
|
|
8
|
+
readonly maxTokens: 1500;
|
|
9
|
+
readonly fallbackOverlap: 0.1;
|
|
10
|
+
readonly enrichment: "path-scope-symbol-v1";
|
|
11
|
+
readonly policy: "source-v1";
|
|
12
|
+
};
|
|
13
|
+
export type Span = {
|
|
14
|
+
startByte: number;
|
|
15
|
+
endByte: number;
|
|
16
|
+
startLine: number;
|
|
17
|
+
endLine: number;
|
|
18
|
+
chunkKind: string;
|
|
19
|
+
scope?: string;
|
|
20
|
+
symbol?: string;
|
|
21
|
+
signature?: string;
|
|
22
|
+
searchText: string;
|
|
23
|
+
tokenCount: number;
|
|
24
|
+
};
|
|
25
|
+
export declare function detectLanguage(path: string): string;
|
|
26
|
+
export declare function chunk(source: string, path: string, mode?: "syntax" | "window"): Promise<{
|
|
27
|
+
spans: Span[];
|
|
28
|
+
parseStatus: string;
|
|
29
|
+
language: string;
|
|
30
|
+
}>;
|