@functional-systems/srcx 0.1.0-dev.38 → 0.1.0-dev.39

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -2,6 +2,14 @@
2
2
 
3
3
  ## Unreleased
4
4
 
5
+ - Add experimental `import --artifact-storage segments` for self-contained,
6
+ digest-verified local format-2 artifacts. Reuse unchanged per-file objects with
7
+ hardlinks and explicit copy fallback; retain format-1 reads and the JSONL default,
8
+ complete source/hash/vector validation, resumable publication and immutable reads.
9
+ Preview JSON adds `artifactFormat`; existing artifacts reject a storage override.
10
+ Measure complete build time, writes and additional inode allocation separately
11
+ from the earlier storage-only prototype; this is not a default promotion.
12
+
5
13
  - Batch local artifact records in a bounded 64 KiB write buffer and hash their
6
14
  existing canonical representation instead of serializing each record twice.
7
15
  Preserve the artifact format, record order/hashes, document limits and full
package/DESIGN.md CHANGED
@@ -460,6 +460,149 @@ record and 20 chunk records; these can share upload requests with other files.
460
460
  Original text appears once in its file record, with exact source ranges in chunks.
461
461
  Search context can repeat in chunk search text without changing the quoted source.
462
462
 
463
+ ### Experimental segmented local artifacts
464
+
465
+ Status: opt-in implementation. `import --artifact-storage segments` writes local
466
+ format 2; the default remains format 1 and a single `records.jsonl`. Presets,
467
+ remote schema, publication ordering and full validation are unchanged. See the
468
+ storage-only prototype and integrated-build measurements in `VALIDATION.md`.
469
+
470
+ The segmented layout uses one logical segment per included file: the exact
471
+ file record followed by all of its chunks, in ordinal order. A new artifact owns
472
+ all objects it needs. It may share physical storage through hardlinks, but never
473
+ depends on a previous artifact's directory, a global cache, or the current Git
474
+ checkout after construction. This targets retained-record rewrites; it does not
475
+ make full validation proportional to the delta.
476
+
477
+ The segmented descriptor extends the local storage version only:
478
+
479
+ ```ts
480
+ type RecordPart = {
481
+ fileId: string;
482
+ digest: string; // SHA-256 of exact canonical UTF-8 JSONL, including newlines.
483
+ byteLength: number;
484
+ recordCount: number;
485
+ };
486
+
487
+ // Existing logical Build fields remain unchanged.
488
+ // `directory` is resolved at load time, never used as an external object pointer.
489
+ type SegmentedBuild = Omit<Build, "format"> & {
490
+ format: 2;
491
+ recordParts: RecordPart[];
492
+ };
493
+ ```
494
+
495
+ Objects live at `objects/<64-lowercase-hex-digest>.jsonl` beneath the artifact.
496
+ Ordered references preserve the builder's existing changed-then-retained record
497
+ order, with no excluded-file segments. Full inventory entries, `recordHashes`,
498
+ counts and aggregate hashes remain in `build.json`. This keeps logical
499
+ identity independent of physical layout: for the same repository/commit/config,
500
+ changing only storage layout retains `buildId`, `configHash`, `recordsHash` and
501
+ `inventoryHash`. Object digests add storage integrity checks; they do not replace
502
+ record/source checks.
503
+
504
+ Require exactly one part per included inventory file, no duplicate file references,
505
+ and `recordCount === 1 + chunkIds.length` for that file. Each object must begin with
506
+ its referenced file record and contain only that file's expected chunks. Byte and
507
+ record counts must be positive safe integers, not allocation instructions from
508
+ untrusted metadata.
509
+
510
+ The writer and full validator follow these rules:
511
+
512
+ 1. Fully validate the previous artifact before reusing its records or hashes.
513
+ Keep the existing repository/configuration and Git blob/mode reuse conditions.
514
+ For a format-2 previous build, find retained objects from its validated file
515
+ inventory and part mapping; avoid the extra whole-record copy pass.
516
+ 2. Stream added/modified file groups to unique temporary objects, with the current
517
+ per-document size checks and bounded write buffer. Hash actual serialized
518
+ bytes, close each object and install it under its digest. Do not buffer an
519
+ entire file's chunk set in memory or impose an arbitrary new segment-size cap.
520
+ 3. Hardlink retained objects into the new artifact, using exclusive destination
521
+ creation. `EXDEV`, `ENOTSUP`, `EOPNOTSUPP` and `ENOSYS` use exclusive copy
522
+ fallback. Other I/O, missing-object or permission failures remain failures.
523
+ No destination reference may point back into the previous directory.
524
+ 4. Keep new output private to its attempt. Finish all objects before installing
525
+ the descriptor, then run complete validation against the new artifact. Object
526
+ write failures remove only that attempt's partial output. Interrupted output
527
+ is never accepted because a descriptor or marker exists. Atomic rename alone
528
+ is not an fsync or power-loss durability guarantee.
529
+ 5. On every full validation, check descriptor shape and logical membership,
530
+ recompute each referenced object's byte digest/count, and run all existing
531
+ uniqueness, record-hash, exact source/position/token and embedding-input checks.
532
+ A valid digest or matching count cannot establish that an old file belongs to
533
+ the new build. Retained records are not sampled or exempted.
534
+ 6. Reject arbitrary object paths, symlink objects/directories and special files.
535
+ Use digest-derived paths, `O_NOFOLLOW | O_NONBLOCK` file opens and post-open
536
+ regular-file/size checks, plus a local regular-directory check. The feature
537
+ requires native no-follow support; the local measurements use macOS. The existing
538
+ single-host model assumes importer-owned artifact directories; hostile
539
+ concurrent directory replacement by the same OS user is outside that model.
540
+ Keep one open record handle and bounded UTF-8 line decoding, retaining the
541
+ existing largest-document allowance. File mode, inode identity, mtime and a
542
+ previous validation receipt are not integrity proofs.
543
+
544
+ The shared `records()` iterator verifies object digests when fully consumed, as
545
+ `validateBuild()` always does. Partial consumers do not establish completeness;
546
+ publication readiness probes follow full build validation. No persistent
547
+ validation receipt skips object/record/source checks on restart.
548
+
549
+ Hardlinks reduce copying but do not enforce immutability: a write to one shared
550
+ inode can corrupt several local artifacts. Rehashing and complete source checks
551
+ must detect this on every use. Read-only modes can be defense in depth only.
552
+ Remote published Tags remain the authority for immutable version reads and do
553
+ not depend on the survival or integrity of local hardlinks.
554
+
555
+ | Condition | Required behavior |
556
+ | --- | --- |
557
+ | Previous directory deleted after the new build | New artifact remains self-contained and validates. |
558
+ | Artifact copied or relocated | Resolve all references beneath the new directory; no original paths needed. |
559
+ | Missing/corrupt object in the pending build | Fail before corpus mutation; retain the journal. Never substitute the current Git tip. |
560
+ | Previous local build entirely absent | Preserve the existing cold-build/remote-baseline recovery paths. |
561
+ | Missing object in an otherwise present previous build | Preserve the current missing-record validation failure; no silent reuse of metadata. |
562
+ | Previous local cache unavailable during publication retry | Validate the intact pending artifact and use its journal's immutable remote baseline, as today. |
563
+ | Corrupt matching local baseline cache | Preserve fail-closed validation rather than trusting its declared hashes. |
564
+ | Cross-filesystem link failure | Explicit copy fallback writes retained bytes too; no delta-write claim for that path. |
565
+ | Partial object/descriptor write or stale local receipt | Remain unpublished; reconstruct only from the exact original build identity or restore matching objects. |
566
+
567
+ No global shared-object garbage collector is needed:
568
+ deleting one artifact unlinks its own objects without invalidating other hardlinks.
569
+ Do not prune local artifacts associated with an active journal or claim this
570
+ implements remote version retention/cleanup.
571
+
572
+ Runtime integration and compatibility:
573
+
574
+ - The storage-format reader/descriptor validator retains legacy format-1 reads
575
+ and resumption. Internal readers accept the already loaded Build, avoiding
576
+ repeated parsing of a large descriptor. Directory-based callers load its
577
+ `build.json`. Unknown formats and missing format-2 descriptors/objects fail;
578
+ they do not fall back to a stray
579
+ `records.jsonl` merely because it exists.
580
+ - `--artifact-storage jsonl|segments` selects storage for a new build and defaults
581
+ to `jsonl`, even when the previous artifact is segmented. Initial conversion
582
+ from format 1 requires the full retained-record pass. Existing `--artifact`
583
+ imports use the recorded format and reject an explicit storage override.
584
+ Preview JSON adds `artifactFormat` (1 or 2); the remaining logical fields and
585
+ exit codes are unchanged. Storage selection is not an embedding preset change.
586
+ - `validateBuild`, previous-record reuse, publication readiness probes and
587
+ changed-record uploads use the shared reader. Full candidate validation,
588
+ final-marker ordering and managed-vector validation/hydration remain intact.
589
+ - Default JSONL evaluation fixtures and frozen archive byte hashes remain format
590
+ 1. Consumers reading `records.jsonl` directly must opt into the shared reader
591
+ before accepting format 2; they are not automatically migrated. Installed-package
592
+ lexical/managed CLI tests cover segmented resumption, loss of the previous local
593
+ artifact, tracked imports and immutable reads.
594
+
595
+ Default promotion requires end-to-end measurements including descriptor bytes, initial
596
+ conversion, full validation, cold recovery, cross-filesystem copies and bounded
597
+ memory/open handles. The prototype reduces record payload writes but adds many
598
+ file operations and shows slower validation. Integrated builds also trade speed
599
+ for materially lower writes and additional allocated storage. This justifies an
600
+ explicit experimental choice, while format 1 remains the default. Pack-based physical
601
+ storage or batched reads can be investigated later if per-file open/link overhead
602
+ dominates; neither is silently assumed by this implementation. Skipping full remote
603
+ candidate verification still requires the separate verified backend contract
604
+ described in section 9. Dependency ingestion remains outside this work.
605
+
463
606
  ## 7. Code-search schema v1
464
607
 
465
608
  The Collection uses one sparse document schema with `kind` distinguishing file,
package/README.md CHANGED
@@ -139,7 +139,39 @@ and tag names that collide require `refs/heads/...` or `refs/tags/...`. No autom
139
139
  fetch, history replay, or revision expressions are supported.
140
140
 
141
141
  Each artifact contains `build.json` (inventory, configuration, counts, hashes,
142
- change/deletion plan) and `records.jsonl` (exact file records and retrieval chunks).
142
+ change/deletion plan) and, by default, `records.jsonl` (exact file records and
143
+ retrieval chunks).
144
+
145
+ For an experimental alternative that reduces retained-record rewrites, use
146
+ `--artifact-storage segments` when creating a new build:
147
+
148
+ ```sh
149
+ srcx import --path /path/to/repo --ref main --dry-run \
150
+ --artifact-storage segments --output /tmp/srcx-segmented-a
151
+ srcx import --path /path/to/repo --ref next --dry-run \
152
+ --artifact-storage segments --previous /tmp/srcx-segmented-a \
153
+ --output /tmp/srcx-segmented-b
154
+ ```
155
+
156
+ Format 2 stores `build.json` and digest-addressed JSONL objects under `objects/`.
157
+ Unchanged objects can share filesystem storage through hardlinks, while each
158
+ artifact remains usable after deleting the previous directory. Full local source
159
+ and record/hash checks still run; publication retains complete candidate
160
+ validation. Fewer bytes are written, but more
161
+ file operations can make builds slower; descriptor construction and full checks
162
+ still scale with corpus size. Segmented mode requires native no-follow and
163
+ nonblocking file-open support. Cross-filesystem/unsupported-link copies rewrite the
164
+ retained bytes too. See [local build measurements](VALIDATION.md).
165
+
166
+ The default is `--artifact-storage jsonl`; storage does not change the embedding
167
+ preset or require a new Collection. Preview JSON includes `artifactFormat` (1 or 2).
168
+ Use `--artifact <directory>` to import/resume either format without a storage flag;
169
+ an explicit `--artifact-storage` cannot override an existing artifact. Older CLIs
170
+ accept only format 1, and tools that directly read `records.jsonl` need the shared
171
+ artifact reader to support format 2. Linked files are not tamper-proof: full checks
172
+ detect shared-object corruption. Preserve the pending artifact and journal until
173
+ publication completes.
174
+
143
175
  Artifacts contain source code; keep them with the same access policy as the repo.
144
176
  Every tree entry is accounted for, including explicit exclusions for symlinks,
145
177
  submodules, LFS pointers, dependencies/build output, image assets (including SVG), binary/invalid UTF-8 data,
package/VALIDATION.md CHANGED
@@ -979,6 +979,71 @@ removing additional write calls. A segmented immutable-artifact design would nee
979
979
  explicit integrity, availability and recovery rules before reusing validation
980
980
  across builds. No local or remote full-validation shortcut is introduced here.
981
981
 
982
+ ## Integrated experimental segmented builds
983
+
984
+ The opt-in runtime now implements `import --artifact-storage segments`; JSONL is
985
+ still the default. Production objects are streamed within the existing
986
+ document/64 KiB buffer bounds,
987
+ opened with native no-follow/nonblocking flags and checked as regular files, and
988
+ fully validated on every build/import. The threat boundary remains importer-owned
989
+ artifact directories and the existing single-host writer model. The directory
990
+ hierarchy is not protected against hostile concurrent replacement by the same
991
+ operating-system user. No object digest or local receipt replaces full source
992
+ checks or complete immutable-candidate verification.
993
+
994
+ An integrated local comparison runs the entire `materialize()` invocation on the
995
+ same fixed 120/600/1,200-file TypeScript corpora, with one addition/modification/
996
+ deletion and three warm repetitions per storage mode. Both modes use the same
997
+ current builder and retain identical record/inventory hashes. A separate compiled
998
+ `1e7fa47` JSONL control checks the original path too. Filesystem observers count
999
+ actual completed record-write bytes and descriptor writes, including `build.json`.
1000
+
1001
+ | Files | JSONL median (ms) | Segments median (ms) | JSONL total write bytes | Segments total write bytes | JSONL record bytes read | Segments record bytes read |
1002
+ | ----- | ----------------- | -------------------- | ----------------------- | -------------------------- | ----------------------- | -------------------------- |
1003
+ | 120 | 215.1 | 256.7 | 3,672,397 | 804,547 | 8,834,926 | 5,889,886 |
1004
+ | 600 | 857.8 | 1,083.0 | 18,340,719 | 3,805,029 | 44,175,406 | 29,450,206 |
1005
+ | 1,200 | 1,664.5 | 2,070.4 | 36,676,122 | 7,555,632 | 88,351,006 | 58,900,606 |
1006
+
1007
+ Host: Node.js 24.15.0, Apple M5 Pro/macOS arm64; warm filesystem/parser state,
1008
+ without fsync. At 1,200 files, record payload writes fall from 29,450,206 to 48,890
1009
+ bytes, while the format-2 descriptor still writes about 7.51 MB. Total application
1010
+ writes fall about 79.4%; wall time rises about 24.4%. The extra retained-record
1011
+ copy pass is eliminated, but previous/new build validation still reads every
1012
+ record. The merged JSONL control's 1,200-file median is 1,623.6 ms, with the same
1013
+ record-byte totals. Timings are host-specific diagnostics, not a general speedup
1014
+ claim or LambdaDB/embedding performance measurement.
1015
+
1016
+ For the 1,200-file update, the sum of `stat.blocks * 512` for newly introduced
1017
+ regular-file inodes falls from 36,679,680 to 7,557,120 bytes. Retained objects share
1018
+ existing inodes with the previous build. This excludes directory blocks and does
1019
+ not resolve filesystem compression, shared extents, snapshots or billed storage;
1020
+ it is an inode-allocation observation, not an exact physical-storage charge.
1021
+
1022
+ Initial conversion from format 1 writes all 29,450,206 record bytes again and
1023
+ retains the full copy pass. A source-identical format-2 build writes zero record
1024
+ bytes, but still writes a roughly 7.50 MB descriptor, links all 1,200 objects and
1025
+ performs both full validations. First-build and conversion measurements are
1026
+ retained with the warm-run evidence. Real cross-filesystem behavior is not tested;
1027
+ an injected `EXDEV` test verifies exclusive copy fallback, while injected `EIO`
1028
+ fails and cleans up. Metadata is still proportional to the corpus.
1029
+
1030
+ Focused regressions cover both conversion directions, relocation/deletion of the
1031
+ previous artifact, actual record/descriptor write counts on a fixed corpus,
1032
+ digest/record/source corruption, stale/duplicate/unsafe references, symlink
1033
+ rejection, large Unicode/CRLF records, early-reader closure, empty builds, unknown
1034
+ formats, argument validation, and partial-write/link cleanup. Managed small/large
1035
+ publication tests resume after old-artifact loss and retain immutable old reads.
1036
+ The installed-tarball suite also exercises segmented lexical/managed CLI retry,
1037
+ tracked imports, search and exact reads through a loopback SDK fixture.
1038
+
1039
+ Validation passed on Node.js 24.15.0 and 22.23.4: 201 full-suite tests, nine
1040
+ installed-tarball CLI tests and typechecking on each runtime. Formatting, version
1041
+ consistency, diff checks and all five CI Python suites (17 tests) also passed.
1042
+ No live backend calls, source uploads, paid operations or package publication
1043
+ were performed for this implementation. Linux/Windows and real
1044
+ cross-volume/power-loss checks have not been run locally; native no-follow support
1045
+ is required for the experimental mode. The default JSONL path is unchanged.
1046
+
982
1047
  ## Local checks
983
1048
 
984
1049
  Runtime used: Node.js 24.15.0, npm 11.12.1. Dependencies are pinned in
@@ -0,0 +1,31 @@
1
+ import { type Doc } from "./common.js";
2
+ import type { Build } from "./build.js";
3
+ export type ArtifactStorage = "jsonl" | "segments";
4
+ export type RecordPart = {
5
+ fileId: string;
6
+ digest: string;
7
+ byteLength: number;
8
+ recordCount: number;
9
+ };
10
+ export declare function validateStorage(b: Build): void;
11
+ /** Digest verification completes when the iterator is exhausted; validateBuild always exhausts it. */
12
+ export declare function artifactRecords(b: Build): AsyncGenerator<Doc>;
13
+ /** Owns only this build's output; retained objects are verified again by full build validation. */
14
+ export declare class ArtifactWriter {
15
+ readonly directory: string;
16
+ readonly storage: ArtifactStorage;
17
+ readonly recordHashes: Record<string, string>;
18
+ readonly parts: RecordPart[];
19
+ private output?;
20
+ private pending;
21
+ private pendingBytes;
22
+ private part?;
23
+ private constructor();
24
+ static create(directory: string, storage: ArtifactStorage): Promise<ArtifactWriter>;
25
+ private flush;
26
+ private finishPart;
27
+ emit(doc: Doc): Promise<void>;
28
+ retain(previous: Build, part: RecordPart, ids: readonly string[]): Promise<void>;
29
+ finish(): Promise<void>;
30
+ close(): Promise<void>;
31
+ }
@@ -0,0 +1,237 @@
1
+ import { constants, createReadStream } from "node:fs";
2
+ import { copyFile, link, lstat, mkdir, open, realpath, rename, } from "node:fs/promises";
3
+ import { createHash, randomUUID } from "node:crypto";
4
+ import { createInterface } from "node:readline";
5
+ import { join } from "node:path";
6
+ import { canonical, hash, invariant } from "./common.js";
7
+ const bufferLimit = 64 * 1024;
8
+ const documentLimit = 4_800_000;
9
+ function requireSegmentSupport() {
10
+ invariant(constants.O_NOFOLLOW && constants.O_NONBLOCK, "Segmented artifacts require no-follow file-open support.");
11
+ }
12
+ const objectPath = (directory, digest) => {
13
+ invariant(/^[a-f0-9]{64}$/.test(digest), "Invalid artifact object digest.");
14
+ return join(directory, "objects", `${digest}.jsonl`);
15
+ };
16
+ export function validateStorage(b) {
17
+ invariant(b.format === 1 || b.format === 2, "Unsupported artifact format.");
18
+ if (b.format === 1)
19
+ return;
20
+ const included = b.inventory.filter((e) => e.status === "included");
21
+ const files = new Map(included.map((e) => [e.fileId, e]));
22
+ invariant(files.size === included.length &&
23
+ Array.isArray(b.recordParts) &&
24
+ b.recordParts.length === included.length, "Invalid segmented artifact inventory.");
25
+ const seen = new Set();
26
+ for (const part of b.recordParts) {
27
+ const entry = files.get(part?.fileId);
28
+ invariant(part &&
29
+ typeof part.fileId === "string" &&
30
+ /^f-[a-f0-9]{64}$/.test(part.fileId) &&
31
+ entry &&
32
+ Array.isArray(entry.chunkIds) &&
33
+ !seen.has(part.fileId) &&
34
+ typeof part.digest === "string" &&
35
+ /^[a-f0-9]{64}$/.test(part.digest) &&
36
+ Number.isSafeInteger(part.byteLength) &&
37
+ part.byteLength > 0 &&
38
+ Number.isSafeInteger(part.recordCount) &&
39
+ part.recordCount === entry.chunkIds.length + 1, "Invalid segmented artifact part.");
40
+ seen.add(part.fileId);
41
+ }
42
+ }
43
+ async function objectDirectory(directory) {
44
+ requireSegmentSupport();
45
+ const path = join(directory, "objects");
46
+ const info = await lstat(path);
47
+ invariant(info.isDirectory() &&
48
+ !info.isSymbolicLink() &&
49
+ (await realpath(path)) === join(await realpath(directory), "objects"), "Artifact objects must be a local regular directory.");
50
+ }
51
+ async function openObject(directory, part) {
52
+ requireSegmentSupport();
53
+ const file = await open(objectPath(directory, part.digest), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
54
+ try {
55
+ const info = await file.stat();
56
+ invariant(info.isFile() && info.size === part.byteLength, "Artifact object is not a regular file of the expected size.");
57
+ return file;
58
+ }
59
+ catch (error) {
60
+ await file.close();
61
+ throw error;
62
+ }
63
+ }
64
+ /** Digest verification completes when the iterator is exhausted; validateBuild always exhausts it. */
65
+ export async function* artifactRecords(b) {
66
+ if (b.format === 1) {
67
+ const input = createReadStream(join(b.directory, "records.jsonl"), {
68
+ encoding: "utf8",
69
+ });
70
+ const lines = createInterface({ input, crlfDelay: Infinity });
71
+ try {
72
+ for await (const line of lines)
73
+ if (line)
74
+ yield JSON.parse(line);
75
+ }
76
+ finally {
77
+ lines.close();
78
+ input.destroy();
79
+ }
80
+ return;
81
+ }
82
+ validateStorage(b);
83
+ await objectDirectory(b.directory);
84
+ const files = new Map(b.inventory.map((e) => [e.fileId, e]));
85
+ for (const part of b.recordParts) {
86
+ const file = await openObject(b.directory, part);
87
+ const input = file.createReadStream({
88
+ autoClose: false,
89
+ highWaterMark: bufferLimit,
90
+ });
91
+ const digest = createHash("sha256"), decoder = new TextDecoder("utf-8", { fatal: true });
92
+ let pending = "", bytes = 0, count = 0;
93
+ try {
94
+ for await (const chunk of input) {
95
+ const data = chunk;
96
+ digest.update(data);
97
+ bytes += data.length;
98
+ pending += decoder.decode(data, { stream: true });
99
+ let newline;
100
+ while ((newline = pending.indexOf("\n")) >= 0) {
101
+ const line = pending.slice(0, newline);
102
+ pending = pending.slice(newline + 1);
103
+ invariant(line.length && Buffer.byteLength(line) <= documentLimit, "Invalid artifact record size.");
104
+ const doc = JSON.parse(line);
105
+ const entry = files.get(part.fileId);
106
+ invariant(count === 0
107
+ ? doc.kind === "file" && doc.id === part.fileId
108
+ : doc.kind === "chunk" &&
109
+ doc.fileId === part.fileId &&
110
+ doc.id === entry.chunkIds[count - 1], "Artifact part membership mismatch.");
111
+ count++;
112
+ yield doc;
113
+ }
114
+ invariant(Buffer.byteLength(pending) <= documentLimit, "Artifact record exceeds safe document limit.");
115
+ }
116
+ pending += decoder.decode();
117
+ invariant(!pending &&
118
+ bytes === part.byteLength &&
119
+ count === part.recordCount &&
120
+ digest.digest("hex") === part.digest, "Artifact object digest/count mismatch.");
121
+ }
122
+ finally {
123
+ input.destroy();
124
+ await file.close();
125
+ }
126
+ }
127
+ }
128
+ /** Owns only this build's output; retained objects are verified again by full build validation. */
129
+ export class ArtifactWriter {
130
+ directory;
131
+ storage;
132
+ recordHashes = {};
133
+ parts = [];
134
+ output;
135
+ pending = [];
136
+ pendingBytes = 0;
137
+ part;
138
+ constructor(directory, storage) {
139
+ this.directory = directory;
140
+ this.storage = storage;
141
+ }
142
+ static async create(directory, storage) {
143
+ const writer = new ArtifactWriter(directory, storage);
144
+ if (storage === "jsonl")
145
+ writer.output = await open(join(directory, "records.jsonl"), "wx", 0o600);
146
+ else {
147
+ requireSegmentSupport();
148
+ await mkdir(join(directory, "objects"), { mode: 0o700 });
149
+ }
150
+ return writer;
151
+ }
152
+ async flush() {
153
+ if (!this.pending.length)
154
+ return;
155
+ await this.output.write(this.pending.join(""));
156
+ this.pending = [];
157
+ this.pendingBytes = 0;
158
+ }
159
+ async finishPart() {
160
+ await this.flush();
161
+ if (!this.part)
162
+ return;
163
+ await this.output.close();
164
+ this.output = undefined;
165
+ const part = this.part;
166
+ const digest = part.digest.digest("hex");
167
+ // Output is private to this newly created artifact; no other writer shares it.
168
+ await rename(part.temp, objectPath(this.directory, digest));
169
+ this.parts.push({
170
+ fileId: part.fileId,
171
+ digest,
172
+ byteLength: part.bytes,
173
+ recordCount: part.count,
174
+ });
175
+ this.part = undefined;
176
+ }
177
+ async emit(doc) {
178
+ const data = canonical(doc), bytes = Buffer.byteLength(data);
179
+ invariant(bytes <= documentLimit, `Serialized document exceeds safe limit: ${doc.id}`);
180
+ invariant(!this.recordHashes[doc.id], "Duplicate document ID.");
181
+ this.recordHashes[doc.id] = hash(data);
182
+ if (this.storage === "segments") {
183
+ if (doc.kind === "file") {
184
+ await this.finishPart();
185
+ const temp = join(this.directory, "objects", `${randomUUID()}.tmp`);
186
+ this.output = await open(temp, "wx", 0o600);
187
+ this.part = {
188
+ fileId: doc.id,
189
+ temp,
190
+ digest: createHash("sha256"),
191
+ bytes: 0,
192
+ count: 0,
193
+ };
194
+ }
195
+ invariant(this.part, "Orphan artifact chunk.");
196
+ this.part.digest.update(data + "\n");
197
+ this.part.bytes += bytes + 1;
198
+ this.part.count++;
199
+ }
200
+ if (this.pendingBytes + bytes + 1 > bufferLimit)
201
+ await this.flush();
202
+ if (bytes + 1 >= bufferLimit)
203
+ await this.output.write(data + "\n");
204
+ else {
205
+ this.pending.push(data + "\n");
206
+ this.pendingBytes += bytes + 1;
207
+ }
208
+ }
209
+ async retain(previous, part, ids) {
210
+ await this.finishPart();
211
+ await objectDirectory(previous.directory);
212
+ const file = await openObject(previous.directory, part);
213
+ await file.close();
214
+ const source = objectPath(previous.directory, part.digest), target = objectPath(this.directory, part.digest);
215
+ try {
216
+ await link(source, target);
217
+ }
218
+ catch (error) {
219
+ if (!["EXDEV", "ENOTSUP", "EOPNOTSUPP", "ENOSYS"].includes(error.code ?? ""))
220
+ throw error;
221
+ await copyFile(source, target, constants.COPYFILE_EXCL);
222
+ }
223
+ for (const id of ids) {
224
+ invariant(!this.recordHashes[id], "Duplicate document ID.");
225
+ this.recordHashes[id] = previous.recordHashes[id];
226
+ }
227
+ this.parts.push({ ...part });
228
+ }
229
+ async finish() {
230
+ await this.finishPart();
231
+ }
232
+ async close() {
233
+ await this.output?.close();
234
+ this.output = undefined;
235
+ }
236
+ }
237
+ //# sourceMappingURL=artifact.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"artifact.js","sourceRoot":"","sources":["../src/artifact.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AACtD,OAAO,EACL,QAAQ,EACR,IAAI,EACJ,KAAK,EACL,KAAK,EACL,IAAI,EACJ,QAAQ,EACR,MAAM,GAEP,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,UAAU,EAAE,UAAU,EAAa,MAAM,aAAa,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAY,MAAM,aAAa,CAAC;AAUnE,MAAM,WAAW,GAAG,EAAE,GAAG,IAAI,CAAC;AAC9B,MAAM,aAAa,GAAG,SAAS,CAAC;AAChC,SAAS,qBAAqB;IAC5B,SAAS,CACP,SAAS,CAAC,UAAU,IAAI,SAAS,CAAC,UAAU,EAC5C,0DAA0D,CAC3D,CAAC;AACJ,CAAC;AACD,MAAM,UAAU,GAAG,CAAC,SAAiB,EAAE,MAAc,EAAE,EAAE;IACvD,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,iCAAiC,CAAC,CAAC;IAC5E,OAAO,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,GAAG,MAAM,QAAQ,CAAC,CAAC;AACvD,CAAC,CAAC;AAEF,MAAM,UAAU,eAAe,CAAC,CAAQ;IACtC,SAAS,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,8BAA8B,CAAC,CAAC;IAC5E,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAC3B,MAAM,QAAQ,GAAG,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC;IACpE,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1D,SAAS,CACP,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,MAAM;QAC5B,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC;QAC5B,CAAC,CAAC,WAAW,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,EAC1C,uCAAuC,CACxC,CAAC;IACF,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACtC,SAAS,CACP,IAAI;YACF,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ;YAC/B,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YACpC,KAAK;YACL,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC;YAC7B,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC;YACtB,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ;YAC/B,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YAClC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC;YACrC,IAAI,CAAC,UAAU,GAAG,CAAC;YACnB,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC;YACtC,IAAI,CAAC,WAAW,KAAK,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAChD,kCAAkC,CACnC,CAAC;QACF,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACxB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,SAAiB;IAC9C,qBAAqB,EAAE,CAAC;IACxB,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,SAAS,CACP,IAAI,CAAC,WAAW,EAAE;QAChB,CAAC,IAAI,CAAC,cAAc,EAAE;QACtB,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,MAAM,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,CAAC,EACvE,qDAAqD,CACtD,CAAC;AACJ,CAAC;AACD,KAAK,UAAU,UAAU,CACvB,SAAiB,EACjB,IAAgB;IAEhB,qBAAqB,EAAE,CAAC;IACxB,MAAM,IAAI,GAAG,MAAM,IAAI,CACrB,UAAU,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,EAClC,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,UAAU,GAAG,SAAS,CAAC,UAAU,CACjE,CAAC;IACF,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAC/B,SAAS,CACP,IAAI,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,UAAU,EAC9C,6DAA6D,CAC9D,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,sGAAsG;AACtG,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,eAAe,CAAC,CAAQ;IAC7C,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnB,MAAM,KAAK,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE;YACjE,QAAQ,EAAE,MAAM;SACjB,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC9D,IAAI,CAAC;YACH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,KAAK;gBAAE,IAAI,IAAI;oBAAE,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAQ,CAAC;QAC1E,CAAC;gBAAS,CAAC;YACT,KAAK,CAAC,KAAK,EAAE,CAAC;YACd,KAAK,CAAC,OAAO,EAAE,CAAC;QAClB,CAAC;QACD,OAAO;IACT,CAAC;IACD,eAAe,CAAC,CAAC,CAAC,CAAC;IACnB,MAAM,eAAe,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACnC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,WAAY,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,CAAC,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC;YAClC,SAAS,EAAE,KAAK;YAChB,aAAa,EAAE,WAAW;SAC3B,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,EACjC,OAAO,GAAG,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,IAAI,OAAO,GAAG,EAAE,EACd,KAAK,GAAG,CAAC,EACT,KAAK,GAAG,CAAC,CAAC;QACZ,IAAI,CAAC;YACH,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;gBAChC,MAAM,IAAI,GAAG,KAAe,CAAC;gBAC7B,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACpB,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC;gBACrB,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;gBAClD,IAAI,OAAe,CAAC;gBACpB,OAAO,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC9C,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;oBACvC,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;oBACrC,SAAS,CACP,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,aAAa,EACvD,+BAA+B,CAChC,CAAC;oBACF,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAQ,CAAC;oBACpC,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAE,CAAC;oBACtC,SAAS,CACP,KAAK,KAAK,CAAC;wBACT,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,MAAM,IAAI,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,MAAM;wBAC/C,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,OAAO;4BAClB,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM;4BAC1B,GAAG,CAAC,EAAE,KAAK,KAAK,CAAC,QAAS,CAAC,KAAK,GAAG,CAAC,CAAC,EAC3C,oCAAoC,CACrC,CAAC;oBACF,KAAK,EAAE,CAAC;oBACR,MAAM,GAAG,CAAC;gBACZ,CAAC;gBACD,SAAS,CACP,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,aAAa,EAC3C,8CAA8C,CAC/C,CAAC;YACJ,CAAC;YACD,OAAO,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YAC5B,SAAS,CACP,CAAC,OAAO;gBACN,KAAK,KAAK,IAAI,CAAC,UAAU;gBACzB,KAAK,KAAK,IAAI,CAAC,WAAW;gBAC1B,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,MAAM,EACtC,wCAAwC,CACzC,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,KAAK,CAAC,OAAO,EAAE,CAAC;YAChB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;AACH,CAAC;AAED,mGAAmG;AACnG,MAAM,OAAO,cAAc;IAcd;IACA;IAdF,YAAY,GAA2B,EAAE,CAAC;IAC1C,KAAK,GAAiB,EAAE,CAAC;IAC1B,MAAM,CAAc;IACpB,OAAO,GAAa,EAAE,CAAC;IACvB,YAAY,GAAG,CAAC,CAAC;IACjB,IAAI,CAMV;IACF,YACW,SAAiB,EACjB,OAAwB;QADxB,cAAS,GAAT,SAAS,CAAQ;QACjB,YAAO,GAAP,OAAO,CAAiB;IAChC,CAAC;IACJ,MAAM,CAAC,KAAK,CAAC,MAAM,CACjB,SAAiB,EACjB,OAAwB;QAExB,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACtD,IAAI,OAAO,KAAK,OAAO;YACrB,MAAM,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;aACvE,CAAC;YACJ,qBAAqB,EAAE,CAAC;YACxB,MAAM,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3D,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IACO,KAAK,CAAC,KAAK;QACjB,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,OAAO;QACjC,MAAM,IAAI,CAAC,MAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAChD,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAClB,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;IACxB,CAAC;IACO,KAAK,CAAC,UAAU;QACtB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,IAAI,CAAC,IAAI;YAAE,OAAO;QACvB,MAAM,IAAI,CAAC,MAAO,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACzC,+EAA+E;QAC/E,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;QAC5D,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;YACd,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM;YACN,UAAU,EAAE,IAAI,CAAC,KAAK;YACtB,WAAW,EAAE,IAAI,CAAC,KAAK;SACxB,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,GAAG,SAAS,CAAC;IACxB,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,GAAQ;QACjB,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,EACzB,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAClC,SAAS,CACP,KAAK,IAAI,aAAa,EACtB,2CAA2C,GAAG,CAAC,EAAE,EAAE,CACpD,CAAC;QACF,SAAS,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,wBAAwB,CAAC,CAAC;QAChE,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,IAAI,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;YAChC,IAAI,GAAG,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACxB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;gBACxB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,GAAG,UAAU,EAAE,MAAM,CAAC,CAAC;gBACpE,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;gBAC5C,IAAI,CAAC,IAAI,GAAG;oBACV,MAAM,EAAE,GAAG,CAAC,EAAE;oBACd,IAAI;oBACJ,MAAM,EAAE,UAAU,CAAC,QAAQ,CAAC;oBAC5B,KAAK,EAAE,CAAC;oBACR,KAAK,EAAE,CAAC;iBACT,CAAC;YACJ,CAAC;YACD,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,wBAAwB,CAAC,CAAC;YAC/C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;YACrC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,KAAK,GAAG,CAAC,CAAC;YAC7B,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QACpB,CAAC;QACD,IAAI,IAAI,CAAC,YAAY,GAAG,KAAK,GAAG,CAAC,GAAG,WAAW;YAAE,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACpE,IAAI,KAAK,GAAG,CAAC,IAAI,WAAW;YAAE,MAAM,IAAI,CAAC,MAAO,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;aAC/D,CAAC;YACJ,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;YAC/B,IAAI,CAAC,YAAY,IAAI,KAAK,GAAG,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IACD,KAAK,CAAC,MAAM,CACV,QAAe,EACf,IAAgB,EAChB,GAAsB;QAEtB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACxB,MAAM,eAAe,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QACxD,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,EACxD,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACnD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC7B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IACE,CAAC,CAAC,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC,QAAQ,CACnD,KAA+B,CAAC,IAAI,IAAI,EAAE,CAC5C;gBAED,MAAM,KAAK,CAAC;YACd,MAAM,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,CAAC,aAAa,CAAC,CAAC;QAC1D,CAAC;QACD,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;YACrB,SAAS,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,EAAE,wBAAwB,CAAC,CAAC;YAC5D,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAE,CAAC;QACrD,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;IAC/B,CAAC;IACD,KAAK,CAAC,MAAM;QACV,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;IAC1B,CAAC;IACD,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;IAC1B,CAAC;CACF"}
package/dist/build.d.ts CHANGED
@@ -1,5 +1,6 @@
1
1
  import { resolveCommit, type Identity } from "./git.js";
2
2
  import { CHUNKER } from "./chunk.js";
3
+ import { type ArtifactStorage, type RecordPart } from "./artifact.js";
3
4
  import { type Doc } from "./common.js";
4
5
  import { type FilePolicy } from "./file-policy.js";
5
6
  export declare const ANALYZERS: readonly ["english", "japanese", "korean", "standard"];
@@ -130,7 +131,8 @@ export type InventoryItem = {
130
131
  skipped?: number;
131
132
  };
132
133
  export type Build = {
133
- format: 1;
134
+ format: 1 | 2;
135
+ recordParts?: RecordPart[];
134
136
  directory: string;
135
137
  repoKey: string;
136
138
  configHash: string;
@@ -158,7 +160,7 @@ export type Embedder = {
158
160
  dimensions: number;
159
161
  embed: (text: string) => Promise<number[]>;
160
162
  };
161
- export declare function records(directory: string): AsyncGenerator<Doc>;
163
+ export declare function records(build: Build | string): AsyncGenerator<Doc>;
162
164
  export declare function loadBuild(directory: string): Promise<Build>;
163
165
  export declare function materialize(args: {
164
166
  identity: Identity;
@@ -166,6 +168,7 @@ export declare function materialize(args: {
166
168
  resolvedCommit?: Awaited<ReturnType<typeof resolveCommit>>;
167
169
  output: string;
168
170
  previous?: Build;
171
+ storage?: ArtifactStorage;
169
172
  preset?: Preset;
170
173
  embedder?: Embedder;
171
174
  cache?: string;