@functional-systems/srcx 0.1.0-dev.38 → 0.1.0-dev.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/DESIGN.md +143 -0
- package/README.md +33 -1
- package/VALIDATION.md +65 -0
- package/dist/artifact.d.ts +31 -0
- package/dist/artifact.js +237 -0
- package/dist/artifact.js.map +1 -0
- package/dist/build.d.ts +5 -2
- package/dist/build.js +48 -56
- package/dist/build.js.map +1 -1
- package/dist/cli.js +7 -1
- package/dist/cli.js.map +1 -1
- package/dist/publish.js +2 -2
- package/dist/publish.js.map +1 -1
- package/package.json +2 -2
- package/src/artifact.ts +303 -0
- package/src/build.ts +55 -54
- package/src/cli.ts +15 -1
- package/src/publish.ts +2 -2
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
## Unreleased
|
|
4
4
|
|
|
5
|
+
- Add experimental `import --artifact-storage segments` for self-contained,
|
|
6
|
+
digest-verified local format-2 artifacts. Reuse unchanged per-file objects with
|
|
7
|
+
hardlinks and explicit copy fallback; retain format-1 reads and the JSONL default,
|
|
8
|
+
complete source/hash/vector validation, resumable publication and immutable reads.
|
|
9
|
+
Preview JSON adds `artifactFormat`; existing artifacts reject a storage override.
|
|
10
|
+
Measure complete build time, writes and additional inode allocation separately
|
|
11
|
+
from the earlier storage-only prototype; this is not a default promotion.
|
|
12
|
+
|
|
5
13
|
- Batch local artifact records in a bounded 64 KiB write buffer and hash their
|
|
6
14
|
existing canonical representation instead of serializing each record twice.
|
|
7
15
|
Preserve the artifact format, record order/hashes, document limits and full
|
package/DESIGN.md
CHANGED
|
@@ -460,6 +460,149 @@ record and 20 chunk records; these can share upload requests with other files.
|
|
|
460
460
|
Original text appears once in its file record, with exact source ranges in chunks.
|
|
461
461
|
Search context can repeat in chunk search text without changing the quoted source.
|
|
462
462
|
|
|
463
|
+
### Experimental segmented local artifacts
|
|
464
|
+
|
|
465
|
+
Status: opt-in implementation. `import --artifact-storage segments` writes local
|
|
466
|
+
format 2; the default remains format 1 and a single `records.jsonl`. Presets,
|
|
467
|
+
remote schema, publication ordering and full validation are unchanged. See the
|
|
468
|
+
storage-only prototype and integrated-build measurements in `VALIDATION.md`.
|
|
469
|
+
|
|
470
|
+
The segmented layout uses one logical segment per included file: the exact
|
|
471
|
+
file record followed by all of its chunks, in ordinal order. A new artifact owns
|
|
472
|
+
all objects it needs. It may share physical storage through hardlinks, but never
|
|
473
|
+
depends on a previous artifact's directory, a global cache, or the current Git
|
|
474
|
+
checkout after construction. This targets retained-record rewrites; it does not
|
|
475
|
+
make full validation proportional to the delta.
|
|
476
|
+
|
|
477
|
+
The segmented descriptor extends the local storage version only:
|
|
478
|
+
|
|
479
|
+
```ts
|
|
480
|
+
type RecordPart = {
|
|
481
|
+
fileId: string;
|
|
482
|
+
digest: string; // SHA-256 of exact canonical UTF-8 JSONL, including newlines.
|
|
483
|
+
byteLength: number;
|
|
484
|
+
recordCount: number;
|
|
485
|
+
};
|
|
486
|
+
|
|
487
|
+
// Existing logical Build fields remain unchanged.
|
|
488
|
+
// `directory` is resolved at load time, never used as an external object pointer.
|
|
489
|
+
type SegmentedBuild = Omit<Build, "format"> & {
|
|
490
|
+
format: 2;
|
|
491
|
+
recordParts: RecordPart[];
|
|
492
|
+
};
|
|
493
|
+
```
|
|
494
|
+
|
|
495
|
+
Objects live at `objects/<64-lowercase-hex-digest>.jsonl` beneath the artifact.
|
|
496
|
+
Ordered references preserve the builder's existing changed-then-retained record
|
|
497
|
+
order, with no excluded-file segments. Full inventory entries, `recordHashes`,
|
|
498
|
+
counts and aggregate hashes remain in `build.json`. This keeps logical
|
|
499
|
+
identity independent of physical layout: for the same repository/commit/config,
|
|
500
|
+
changing only storage layout retains `buildId`, `configHash`, `recordsHash` and
|
|
501
|
+
`inventoryHash`. Object digests add storage integrity checks; they do not replace
|
|
502
|
+
record/source checks.
|
|
503
|
+
|
|
504
|
+
Require exactly one part per included inventory file, no duplicate file references,
|
|
505
|
+
and `recordCount === 1 + chunkIds.length` for that file. Each object must begin with
|
|
506
|
+
its referenced file record and contain only that file's expected chunks. Byte and
|
|
507
|
+
record counts must be positive safe integers, not allocation instructions from
|
|
508
|
+
untrusted metadata.
|
|
509
|
+
|
|
510
|
+
The writer and full validator follow these rules:
|
|
511
|
+
|
|
512
|
+
1. Fully validate the previous artifact before reusing its records or hashes.
|
|
513
|
+
Keep the existing repository/configuration and Git blob/mode reuse conditions.
|
|
514
|
+
For a format-2 previous build, find retained objects from its validated file
|
|
515
|
+
inventory and part mapping; avoid the extra whole-record copy pass.
|
|
516
|
+
2. Stream added/modified file groups to unique temporary objects, with the current
|
|
517
|
+
per-document size checks and bounded write buffer. Hash actual serialized
|
|
518
|
+
bytes, close each object and install it under its digest. Do not buffer an
|
|
519
|
+
entire file's chunk set in memory or impose an arbitrary new segment-size cap.
|
|
520
|
+
3. Hardlink retained objects into the new artifact, using exclusive destination
|
|
521
|
+
creation. `EXDEV`, `ENOTSUP`, `EOPNOTSUPP` and `ENOSYS` use exclusive copy
|
|
522
|
+
fallback. Other I/O, missing-object or permission failures remain failures.
|
|
523
|
+
No destination reference may point back into the previous directory.
|
|
524
|
+
4. Keep new output private to its attempt. Finish all objects before installing
|
|
525
|
+
the descriptor, then run complete validation against the new artifact. Object
|
|
526
|
+
write failures remove only that attempt's partial output. Interrupted output
|
|
527
|
+
is never accepted because a descriptor or marker exists. Atomic rename alone
|
|
528
|
+
is not an fsync or power-loss durability guarantee.
|
|
529
|
+
5. On every full validation, check descriptor shape and logical membership,
|
|
530
|
+
recompute each referenced object's byte digest/count, and run all existing
|
|
531
|
+
uniqueness, record-hash, exact source/position/token and embedding-input checks.
|
|
532
|
+
A valid digest or matching count cannot establish that an old file belongs to
|
|
533
|
+
the new build. Retained records are not sampled or exempted.
|
|
534
|
+
6. Reject arbitrary object paths, symlink objects/directories and special files.
|
|
535
|
+
Use digest-derived paths, `O_NOFOLLOW | O_NONBLOCK` file opens and post-open
|
|
536
|
+
regular-file/size checks, plus a local regular-directory check. The feature
|
|
537
|
+
requires native no-follow support; the local measurements use macOS. The existing
|
|
538
|
+
single-host model assumes importer-owned artifact directories; hostile
|
|
539
|
+
concurrent directory replacement by the same OS user is outside that model.
|
|
540
|
+
Keep one open record handle and bounded UTF-8 line decoding, retaining the
|
|
541
|
+
existing largest-document allowance. File mode, inode identity, mtime and a
|
|
542
|
+
previous validation receipt are not integrity proofs.
|
|
543
|
+
|
|
544
|
+
The shared `records()` iterator verifies object digests when fully consumed, as
|
|
545
|
+
`validateBuild()` always does. Partial consumers do not establish completeness;
|
|
546
|
+
publication readiness probes follow full build validation. No persistent
|
|
547
|
+
validation receipt skips object/record/source checks on restart.
|
|
548
|
+
|
|
549
|
+
Hardlinks reduce copying but do not enforce immutability: a write to one shared
|
|
550
|
+
inode can corrupt several local artifacts. Rehashing and complete source checks
|
|
551
|
+
must detect this on every use. Read-only modes can be defense in depth only.
|
|
552
|
+
Remote published Tags remain the authority for immutable version reads and do
|
|
553
|
+
not depend on the survival or integrity of local hardlinks.
|
|
554
|
+
|
|
555
|
+
| Condition | Required behavior |
|
|
556
|
+
| --- | --- |
|
|
557
|
+
| Previous directory deleted after the new build | New artifact remains self-contained and validates. |
|
|
558
|
+
| Artifact copied or relocated | Resolve all references beneath the new directory; no original paths needed. |
|
|
559
|
+
| Missing/corrupt object in the pending build | Fail before corpus mutation; retain the journal. Never substitute the current Git tip. |
|
|
560
|
+
| Previous local build entirely absent | Preserve the existing cold-build/remote-baseline recovery paths. |
|
|
561
|
+
| Missing object in an otherwise present previous build | Preserve the current missing-record validation failure; no silent reuse of metadata. |
|
|
562
|
+
| Previous local cache unavailable during publication retry | Validate the intact pending artifact and use its journal's immutable remote baseline, as today. |
|
|
563
|
+
| Corrupt matching local baseline cache | Preserve fail-closed validation rather than trusting its declared hashes. |
|
|
564
|
+
| Cross-filesystem link failure | Explicit copy fallback writes retained bytes too; no delta-write claim for that path. |
|
|
565
|
+
| Partial object/descriptor write or stale local receipt | Remain unpublished; reconstruct only from the exact original build identity or restore matching objects. |
|
|
566
|
+
|
|
567
|
+
No global shared-object garbage collector is needed:
|
|
568
|
+
deleting one artifact unlinks its own objects without invalidating other hardlinks.
|
|
569
|
+
Do not prune local artifacts associated with an active journal or claim this
|
|
570
|
+
implements remote version retention/cleanup.
|
|
571
|
+
|
|
572
|
+
Runtime integration and compatibility:
|
|
573
|
+
|
|
574
|
+
- The storage-format reader/descriptor validator retains legacy format-1 reads
|
|
575
|
+
and resumption. Internal readers accept the already loaded Build, avoiding
|
|
576
|
+
repeated parsing of a large descriptor. Directory-based callers load its
|
|
577
|
+
`build.json`. Unknown formats and missing format-2 descriptors/objects fail;
|
|
578
|
+
they do not fall back to a stray
|
|
579
|
+
`records.jsonl` merely because it exists.
|
|
580
|
+
- `--artifact-storage jsonl|segments` selects storage for a new build and defaults
|
|
581
|
+
to `jsonl`, even when the previous artifact is segmented. Initial conversion
|
|
582
|
+
from format 1 requires the full retained-record pass. Existing `--artifact`
|
|
583
|
+
imports use the recorded format and reject an explicit storage override.
|
|
584
|
+
Preview JSON adds `artifactFormat` (1 or 2); the remaining logical fields and
|
|
585
|
+
exit codes are unchanged. Storage selection is not an embedding preset change.
|
|
586
|
+
- `validateBuild`, previous-record reuse, publication readiness probes and
|
|
587
|
+
changed-record uploads use the shared reader. Full candidate validation,
|
|
588
|
+
final-marker ordering and managed-vector validation/hydration remain intact.
|
|
589
|
+
- Default JSONL evaluation fixtures and frozen archive byte hashes remain format
|
|
590
|
+
1. Consumers reading `records.jsonl` directly must opt into the shared reader
|
|
591
|
+
before accepting format 2; they are not automatically migrated. Installed-package
|
|
592
|
+
lexical/managed CLI tests cover segmented resumption, loss of the previous local
|
|
593
|
+
artifact, tracked imports and immutable reads.
|
|
594
|
+
|
|
595
|
+
Default promotion requires end-to-end measurements including descriptor bytes, initial
|
|
596
|
+
conversion, full validation, cold recovery, cross-filesystem copies and bounded
|
|
597
|
+
memory/open handles. The prototype reduces record payload writes but adds many
|
|
598
|
+
file operations and shows slower validation. Integrated builds also trade speed
|
|
599
|
+
for materially lower writes and additional allocated storage. This justifies an
|
|
600
|
+
explicit experimental choice, while format 1 remains the default. Pack-based physical
|
|
601
|
+
storage or batched reads can be investigated later if per-file open/link overhead
|
|
602
|
+
dominates; neither is silently assumed by this implementation. Skipping full remote
|
|
603
|
+
candidate verification still requires the separate verified backend contract
|
|
604
|
+
described in section 9. Dependency ingestion remains outside this work.
|
|
605
|
+
|
|
463
606
|
## 7. Code-search schema v1
|
|
464
607
|
|
|
465
608
|
The Collection uses one sparse document schema with `kind` distinguishing file,
|
package/README.md
CHANGED
|
@@ -139,7 +139,39 @@ and tag names that collide require `refs/heads/...` or `refs/tags/...`. No autom
|
|
|
139
139
|
fetch, history replay, or revision expressions are supported.
|
|
140
140
|
|
|
141
141
|
Each artifact contains `build.json` (inventory, configuration, counts, hashes,
|
|
142
|
-
change/deletion plan) and `records.jsonl` (exact file records and
|
|
142
|
+
change/deletion plan) and, by default, `records.jsonl` (exact file records and
|
|
143
|
+
retrieval chunks).
|
|
144
|
+
|
|
145
|
+
For an experimental alternative that reduces retained-record rewrites, use
|
|
146
|
+
`--artifact-storage segments` when creating a new build:
|
|
147
|
+
|
|
148
|
+
```sh
|
|
149
|
+
srcx import --path /path/to/repo --ref main --dry-run \
|
|
150
|
+
--artifact-storage segments --output /tmp/srcx-segmented-a
|
|
151
|
+
srcx import --path /path/to/repo --ref next --dry-run \
|
|
152
|
+
--artifact-storage segments --previous /tmp/srcx-segmented-a \
|
|
153
|
+
--output /tmp/srcx-segmented-b
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
Format 2 stores `build.json` and digest-addressed JSONL objects under `objects/`.
|
|
157
|
+
Unchanged objects can share filesystem storage through hardlinks, while each
|
|
158
|
+
artifact remains usable after deleting the previous directory. Full local source
|
|
159
|
+
and record/hash checks still run; publication retains complete candidate
|
|
160
|
+
validation. Fewer bytes are written, but more
|
|
161
|
+
file operations can make builds slower; descriptor construction and full checks
|
|
162
|
+
still scale with corpus size. Segmented mode requires native no-follow and
|
|
163
|
+
nonblocking file-open support. Cross-filesystem/unsupported-link copies rewrite the
|
|
164
|
+
retained bytes too. See [local build measurements](VALIDATION.md).
|
|
165
|
+
|
|
166
|
+
The default is `--artifact-storage jsonl`; storage does not change the embedding
|
|
167
|
+
preset or require a new Collection. Preview JSON includes `artifactFormat` (1 or 2).
|
|
168
|
+
Use `--artifact <directory>` to import/resume either format without a storage flag;
|
|
169
|
+
an explicit `--artifact-storage` cannot override an existing artifact. Older CLIs
|
|
170
|
+
accept only format 1, and tools that directly read `records.jsonl` need the shared
|
|
171
|
+
artifact reader to support format 2. Linked files are not tamper-proof: full checks
|
|
172
|
+
detect shared-object corruption. Preserve the pending artifact and journal until
|
|
173
|
+
publication completes.
|
|
174
|
+
|
|
143
175
|
Artifacts contain source code; keep them with the same access policy as the repo.
|
|
144
176
|
Every tree entry is accounted for, including explicit exclusions for symlinks,
|
|
145
177
|
submodules, LFS pointers, dependencies/build output, image assets (including SVG), binary/invalid UTF-8 data,
|
package/VALIDATION.md
CHANGED
|
@@ -979,6 +979,71 @@ removing additional write calls. A segmented immutable-artifact design would nee
|
|
|
979
979
|
explicit integrity, availability and recovery rules before reusing validation
|
|
980
980
|
across builds. No local or remote full-validation shortcut is introduced here.
|
|
981
981
|
|
|
982
|
+
## Integrated experimental segmented builds
|
|
983
|
+
|
|
984
|
+
The opt-in runtime now implements `import --artifact-storage segments`; JSONL is
|
|
985
|
+
still the default. Production objects are streamed within the existing
|
|
986
|
+
document/64 KiB buffer bounds,
|
|
987
|
+
opened with native no-follow/nonblocking flags and checked as regular files, and
|
|
988
|
+
fully validated on every build/import. The threat boundary remains importer-owned
|
|
989
|
+
artifact directories and the existing single-host writer model. The directory
|
|
990
|
+
hierarchy is not protected against hostile concurrent replacement by the same
|
|
991
|
+
operating-system user. No object digest or local receipt replaces full source
|
|
992
|
+
checks or complete immutable-candidate verification.
|
|
993
|
+
|
|
994
|
+
An integrated local comparison runs the entire `materialize()` invocation on the
|
|
995
|
+
same fixed 120/600/1,200-file TypeScript corpora, with one addition/modification/
|
|
996
|
+
deletion and three warm repetitions per storage mode. Both modes use the same
|
|
997
|
+
current builder and retain identical record/inventory hashes. A separate compiled
|
|
998
|
+
`1e7fa47` JSONL control checks the original path too. Filesystem observers count
|
|
999
|
+
actual completed record-write bytes and descriptor writes, including `build.json`.
|
|
1000
|
+
|
|
1001
|
+
| Files | JSONL median (ms) | Segments median (ms) | JSONL total write bytes | Segments total write bytes | JSONL record bytes read | Segments record bytes read |
|
|
1002
|
+
| ----- | ----------------- | -------------------- | ----------------------- | -------------------------- | ----------------------- | -------------------------- |
|
|
1003
|
+
| 120 | 215.1 | 256.7 | 3,672,397 | 804,547 | 8,834,926 | 5,889,886 |
|
|
1004
|
+
| 600 | 857.8 | 1,083.0 | 18,340,719 | 3,805,029 | 44,175,406 | 29,450,206 |
|
|
1005
|
+
| 1,200 | 1,664.5 | 2,070.4 | 36,676,122 | 7,555,632 | 88,351,006 | 58,900,606 |
|
|
1006
|
+
|
|
1007
|
+
Host: Node.js 24.15.0, Apple M5 Pro/macOS arm64; warm filesystem/parser state,
|
|
1008
|
+
without fsync. At 1,200 files, record payload writes fall from 29,450,206 to 48,890
|
|
1009
|
+
bytes, while the format-2 descriptor still writes about 7.51 MB. Total application
|
|
1010
|
+
writes fall about 79.4%; wall time rises about 24.4%. The extra retained-record
|
|
1011
|
+
copy pass is eliminated, but previous/new build validation still reads every
|
|
1012
|
+
record. The merged JSONL control's 1,200-file median is 1,623.6 ms, with the same
|
|
1013
|
+
record-byte totals. Timings are host-specific diagnostics, not a general speedup
|
|
1014
|
+
claim or LambdaDB/embedding performance measurement.
|
|
1015
|
+
|
|
1016
|
+
For the 1,200-file update, the sum of `stat.blocks * 512` for newly introduced
|
|
1017
|
+
regular-file inodes falls from 36,679,680 to 7,557,120 bytes. Retained objects share
|
|
1018
|
+
existing inodes with the previous build. This excludes directory blocks and does
|
|
1019
|
+
not resolve filesystem compression, shared extents, snapshots or billed storage;
|
|
1020
|
+
it is an inode-allocation observation, not an exact physical-storage charge.
|
|
1021
|
+
|
|
1022
|
+
Initial conversion from format 1 writes all 29,450,206 record bytes again and
|
|
1023
|
+
retains the full copy pass. A source-identical format-2 build writes zero record
|
|
1024
|
+
bytes, but still writes a roughly 7.50 MB descriptor, links all 1,200 objects and
|
|
1025
|
+
performs both full validations. First-build and conversion measurements are
|
|
1026
|
+
retained with the warm-run evidence. Real cross-filesystem behavior is not tested;
|
|
1027
|
+
an injected `EXDEV` test verifies exclusive copy fallback, while injected `EIO`
|
|
1028
|
+
fails and cleans up. Metadata is still proportional to the corpus.
|
|
1029
|
+
|
|
1030
|
+
Focused regressions cover both conversion directions, relocation/deletion of the
|
|
1031
|
+
previous artifact, actual record/descriptor write counts on a fixed corpus,
|
|
1032
|
+
digest/record/source corruption, stale/duplicate/unsafe references, symlink
|
|
1033
|
+
rejection, large Unicode/CRLF records, early-reader closure, empty builds, unknown
|
|
1034
|
+
formats, argument validation, and partial-write/link cleanup. Managed small/large
|
|
1035
|
+
publication tests resume after old-artifact loss and retain immutable old reads.
|
|
1036
|
+
The installed-tarball suite also exercises segmented lexical/managed CLI retry,
|
|
1037
|
+
tracked imports, search and exact reads through a loopback SDK fixture.
|
|
1038
|
+
|
|
1039
|
+
Validation passed on Node.js 24.15.0 and 22.23.4: 201 full-suite tests, nine
|
|
1040
|
+
installed-tarball CLI tests and typechecking on each runtime. Formatting, version
|
|
1041
|
+
consistency, diff checks and all five CI Python suites (17 tests) also passed.
|
|
1042
|
+
No live backend calls, source uploads, paid operations or package publication
|
|
1043
|
+
were performed for this implementation. Linux/Windows and real
|
|
1044
|
+
cross-volume/power-loss checks have not been run locally; native no-follow support
|
|
1045
|
+
is required for the experimental mode. The default JSONL path is unchanged.
|
|
1046
|
+
|
|
982
1047
|
## Local checks
|
|
983
1048
|
|
|
984
1049
|
Runtime used: Node.js 24.15.0, npm 11.12.1. Dependencies are pinned in
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { type Doc } from "./common.js";
|
|
2
|
+
import type { Build } from "./build.js";
|
|
3
|
+
export type ArtifactStorage = "jsonl" | "segments";
|
|
4
|
+
export type RecordPart = {
|
|
5
|
+
fileId: string;
|
|
6
|
+
digest: string;
|
|
7
|
+
byteLength: number;
|
|
8
|
+
recordCount: number;
|
|
9
|
+
};
|
|
10
|
+
export declare function validateStorage(b: Build): void;
|
|
11
|
+
/** Digest verification completes when the iterator is exhausted; validateBuild always exhausts it. */
|
|
12
|
+
export declare function artifactRecords(b: Build): AsyncGenerator<Doc>;
|
|
13
|
+
/** Owns only this build's output; retained objects are verified again by full build validation. */
|
|
14
|
+
export declare class ArtifactWriter {
|
|
15
|
+
readonly directory: string;
|
|
16
|
+
readonly storage: ArtifactStorage;
|
|
17
|
+
readonly recordHashes: Record<string, string>;
|
|
18
|
+
readonly parts: RecordPart[];
|
|
19
|
+
private output?;
|
|
20
|
+
private pending;
|
|
21
|
+
private pendingBytes;
|
|
22
|
+
private part?;
|
|
23
|
+
private constructor();
|
|
24
|
+
static create(directory: string, storage: ArtifactStorage): Promise<ArtifactWriter>;
|
|
25
|
+
private flush;
|
|
26
|
+
private finishPart;
|
|
27
|
+
emit(doc: Doc): Promise<void>;
|
|
28
|
+
retain(previous: Build, part: RecordPart, ids: readonly string[]): Promise<void>;
|
|
29
|
+
finish(): Promise<void>;
|
|
30
|
+
close(): Promise<void>;
|
|
31
|
+
}
|
package/dist/artifact.js
ADDED
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
import { constants, createReadStream } from "node:fs";
|
|
2
|
+
import { copyFile, link, lstat, mkdir, open, realpath, rename, } from "node:fs/promises";
|
|
3
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
4
|
+
import { createInterface } from "node:readline";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { canonical, hash, invariant } from "./common.js";
|
|
7
|
+
const bufferLimit = 64 * 1024;
|
|
8
|
+
const documentLimit = 4_800_000;
|
|
9
|
+
function requireSegmentSupport() {
|
|
10
|
+
invariant(constants.O_NOFOLLOW && constants.O_NONBLOCK, "Segmented artifacts require no-follow file-open support.");
|
|
11
|
+
}
|
|
12
|
+
const objectPath = (directory, digest) => {
|
|
13
|
+
invariant(/^[a-f0-9]{64}$/.test(digest), "Invalid artifact object digest.");
|
|
14
|
+
return join(directory, "objects", `${digest}.jsonl`);
|
|
15
|
+
};
|
|
16
|
+
export function validateStorage(b) {
|
|
17
|
+
invariant(b.format === 1 || b.format === 2, "Unsupported artifact format.");
|
|
18
|
+
if (b.format === 1)
|
|
19
|
+
return;
|
|
20
|
+
const included = b.inventory.filter((e) => e.status === "included");
|
|
21
|
+
const files = new Map(included.map((e) => [e.fileId, e]));
|
|
22
|
+
invariant(files.size === included.length &&
|
|
23
|
+
Array.isArray(b.recordParts) &&
|
|
24
|
+
b.recordParts.length === included.length, "Invalid segmented artifact inventory.");
|
|
25
|
+
const seen = new Set();
|
|
26
|
+
for (const part of b.recordParts) {
|
|
27
|
+
const entry = files.get(part?.fileId);
|
|
28
|
+
invariant(part &&
|
|
29
|
+
typeof part.fileId === "string" &&
|
|
30
|
+
/^f-[a-f0-9]{64}$/.test(part.fileId) &&
|
|
31
|
+
entry &&
|
|
32
|
+
Array.isArray(entry.chunkIds) &&
|
|
33
|
+
!seen.has(part.fileId) &&
|
|
34
|
+
typeof part.digest === "string" &&
|
|
35
|
+
/^[a-f0-9]{64}$/.test(part.digest) &&
|
|
36
|
+
Number.isSafeInteger(part.byteLength) &&
|
|
37
|
+
part.byteLength > 0 &&
|
|
38
|
+
Number.isSafeInteger(part.recordCount) &&
|
|
39
|
+
part.recordCount === entry.chunkIds.length + 1, "Invalid segmented artifact part.");
|
|
40
|
+
seen.add(part.fileId);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
async function objectDirectory(directory) {
|
|
44
|
+
requireSegmentSupport();
|
|
45
|
+
const path = join(directory, "objects");
|
|
46
|
+
const info = await lstat(path);
|
|
47
|
+
invariant(info.isDirectory() &&
|
|
48
|
+
!info.isSymbolicLink() &&
|
|
49
|
+
(await realpath(path)) === join(await realpath(directory), "objects"), "Artifact objects must be a local regular directory.");
|
|
50
|
+
}
|
|
51
|
+
async function openObject(directory, part) {
|
|
52
|
+
requireSegmentSupport();
|
|
53
|
+
const file = await open(objectPath(directory, part.digest), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
54
|
+
try {
|
|
55
|
+
const info = await file.stat();
|
|
56
|
+
invariant(info.isFile() && info.size === part.byteLength, "Artifact object is not a regular file of the expected size.");
|
|
57
|
+
return file;
|
|
58
|
+
}
|
|
59
|
+
catch (error) {
|
|
60
|
+
await file.close();
|
|
61
|
+
throw error;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
/** Digest verification completes when the iterator is exhausted; validateBuild always exhausts it. */
|
|
65
|
+
export async function* artifactRecords(b) {
|
|
66
|
+
if (b.format === 1) {
|
|
67
|
+
const input = createReadStream(join(b.directory, "records.jsonl"), {
|
|
68
|
+
encoding: "utf8",
|
|
69
|
+
});
|
|
70
|
+
const lines = createInterface({ input, crlfDelay: Infinity });
|
|
71
|
+
try {
|
|
72
|
+
for await (const line of lines)
|
|
73
|
+
if (line)
|
|
74
|
+
yield JSON.parse(line);
|
|
75
|
+
}
|
|
76
|
+
finally {
|
|
77
|
+
lines.close();
|
|
78
|
+
input.destroy();
|
|
79
|
+
}
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
validateStorage(b);
|
|
83
|
+
await objectDirectory(b.directory);
|
|
84
|
+
const files = new Map(b.inventory.map((e) => [e.fileId, e]));
|
|
85
|
+
for (const part of b.recordParts) {
|
|
86
|
+
const file = await openObject(b.directory, part);
|
|
87
|
+
const input = file.createReadStream({
|
|
88
|
+
autoClose: false,
|
|
89
|
+
highWaterMark: bufferLimit,
|
|
90
|
+
});
|
|
91
|
+
const digest = createHash("sha256"), decoder = new TextDecoder("utf-8", { fatal: true });
|
|
92
|
+
let pending = "", bytes = 0, count = 0;
|
|
93
|
+
try {
|
|
94
|
+
for await (const chunk of input) {
|
|
95
|
+
const data = chunk;
|
|
96
|
+
digest.update(data);
|
|
97
|
+
bytes += data.length;
|
|
98
|
+
pending += decoder.decode(data, { stream: true });
|
|
99
|
+
let newline;
|
|
100
|
+
while ((newline = pending.indexOf("\n")) >= 0) {
|
|
101
|
+
const line = pending.slice(0, newline);
|
|
102
|
+
pending = pending.slice(newline + 1);
|
|
103
|
+
invariant(line.length && Buffer.byteLength(line) <= documentLimit, "Invalid artifact record size.");
|
|
104
|
+
const doc = JSON.parse(line);
|
|
105
|
+
const entry = files.get(part.fileId);
|
|
106
|
+
invariant(count === 0
|
|
107
|
+
? doc.kind === "file" && doc.id === part.fileId
|
|
108
|
+
: doc.kind === "chunk" &&
|
|
109
|
+
doc.fileId === part.fileId &&
|
|
110
|
+
doc.id === entry.chunkIds[count - 1], "Artifact part membership mismatch.");
|
|
111
|
+
count++;
|
|
112
|
+
yield doc;
|
|
113
|
+
}
|
|
114
|
+
invariant(Buffer.byteLength(pending) <= documentLimit, "Artifact record exceeds safe document limit.");
|
|
115
|
+
}
|
|
116
|
+
pending += decoder.decode();
|
|
117
|
+
invariant(!pending &&
|
|
118
|
+
bytes === part.byteLength &&
|
|
119
|
+
count === part.recordCount &&
|
|
120
|
+
digest.digest("hex") === part.digest, "Artifact object digest/count mismatch.");
|
|
121
|
+
}
|
|
122
|
+
finally {
|
|
123
|
+
input.destroy();
|
|
124
|
+
await file.close();
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
/** Owns only this build's output; retained objects are verified again by full build validation. */
|
|
129
|
+
export class ArtifactWriter {
|
|
130
|
+
directory;
|
|
131
|
+
storage;
|
|
132
|
+
recordHashes = {};
|
|
133
|
+
parts = [];
|
|
134
|
+
output;
|
|
135
|
+
pending = [];
|
|
136
|
+
pendingBytes = 0;
|
|
137
|
+
part;
|
|
138
|
+
constructor(directory, storage) {
|
|
139
|
+
this.directory = directory;
|
|
140
|
+
this.storage = storage;
|
|
141
|
+
}
|
|
142
|
+
static async create(directory, storage) {
|
|
143
|
+
const writer = new ArtifactWriter(directory, storage);
|
|
144
|
+
if (storage === "jsonl")
|
|
145
|
+
writer.output = await open(join(directory, "records.jsonl"), "wx", 0o600);
|
|
146
|
+
else {
|
|
147
|
+
requireSegmentSupport();
|
|
148
|
+
await mkdir(join(directory, "objects"), { mode: 0o700 });
|
|
149
|
+
}
|
|
150
|
+
return writer;
|
|
151
|
+
}
|
|
152
|
+
async flush() {
|
|
153
|
+
if (!this.pending.length)
|
|
154
|
+
return;
|
|
155
|
+
await this.output.write(this.pending.join(""));
|
|
156
|
+
this.pending = [];
|
|
157
|
+
this.pendingBytes = 0;
|
|
158
|
+
}
|
|
159
|
+
async finishPart() {
|
|
160
|
+
await this.flush();
|
|
161
|
+
if (!this.part)
|
|
162
|
+
return;
|
|
163
|
+
await this.output.close();
|
|
164
|
+
this.output = undefined;
|
|
165
|
+
const part = this.part;
|
|
166
|
+
const digest = part.digest.digest("hex");
|
|
167
|
+
// Output is private to this newly created artifact; no other writer shares it.
|
|
168
|
+
await rename(part.temp, objectPath(this.directory, digest));
|
|
169
|
+
this.parts.push({
|
|
170
|
+
fileId: part.fileId,
|
|
171
|
+
digest,
|
|
172
|
+
byteLength: part.bytes,
|
|
173
|
+
recordCount: part.count,
|
|
174
|
+
});
|
|
175
|
+
this.part = undefined;
|
|
176
|
+
}
|
|
177
|
+
async emit(doc) {
|
|
178
|
+
const data = canonical(doc), bytes = Buffer.byteLength(data);
|
|
179
|
+
invariant(bytes <= documentLimit, `Serialized document exceeds safe limit: ${doc.id}`);
|
|
180
|
+
invariant(!this.recordHashes[doc.id], "Duplicate document ID.");
|
|
181
|
+
this.recordHashes[doc.id] = hash(data);
|
|
182
|
+
if (this.storage === "segments") {
|
|
183
|
+
if (doc.kind === "file") {
|
|
184
|
+
await this.finishPart();
|
|
185
|
+
const temp = join(this.directory, "objects", `${randomUUID()}.tmp`);
|
|
186
|
+
this.output = await open(temp, "wx", 0o600);
|
|
187
|
+
this.part = {
|
|
188
|
+
fileId: doc.id,
|
|
189
|
+
temp,
|
|
190
|
+
digest: createHash("sha256"),
|
|
191
|
+
bytes: 0,
|
|
192
|
+
count: 0,
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
invariant(this.part, "Orphan artifact chunk.");
|
|
196
|
+
this.part.digest.update(data + "\n");
|
|
197
|
+
this.part.bytes += bytes + 1;
|
|
198
|
+
this.part.count++;
|
|
199
|
+
}
|
|
200
|
+
if (this.pendingBytes + bytes + 1 > bufferLimit)
|
|
201
|
+
await this.flush();
|
|
202
|
+
if (bytes + 1 >= bufferLimit)
|
|
203
|
+
await this.output.write(data + "\n");
|
|
204
|
+
else {
|
|
205
|
+
this.pending.push(data + "\n");
|
|
206
|
+
this.pendingBytes += bytes + 1;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
async retain(previous, part, ids) {
|
|
210
|
+
await this.finishPart();
|
|
211
|
+
await objectDirectory(previous.directory);
|
|
212
|
+
const file = await openObject(previous.directory, part);
|
|
213
|
+
await file.close();
|
|
214
|
+
const source = objectPath(previous.directory, part.digest), target = objectPath(this.directory, part.digest);
|
|
215
|
+
try {
|
|
216
|
+
await link(source, target);
|
|
217
|
+
}
|
|
218
|
+
catch (error) {
|
|
219
|
+
if (!["EXDEV", "ENOTSUP", "EOPNOTSUPP", "ENOSYS"].includes(error.code ?? ""))
|
|
220
|
+
throw error;
|
|
221
|
+
await copyFile(source, target, constants.COPYFILE_EXCL);
|
|
222
|
+
}
|
|
223
|
+
for (const id of ids) {
|
|
224
|
+
invariant(!this.recordHashes[id], "Duplicate document ID.");
|
|
225
|
+
this.recordHashes[id] = previous.recordHashes[id];
|
|
226
|
+
}
|
|
227
|
+
this.parts.push({ ...part });
|
|
228
|
+
}
|
|
229
|
+
async finish() {
|
|
230
|
+
await this.finishPart();
|
|
231
|
+
}
|
|
232
|
+
async close() {
|
|
233
|
+
await this.output?.close();
|
|
234
|
+
this.output = undefined;
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
//# sourceMappingURL=artifact.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"artifact.js","sourceRoot":"","sources":["../src/artifact.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AACtD,OAAO,EACL,QAAQ,EACR,IAAI,EACJ,KAAK,EACL,KAAK,EACL,IAAI,EACJ,QAAQ,EACR,MAAM,GAEP,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,UAAU,EAAE,UAAU,EAAa,MAAM,aAAa,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAY,MAAM,aAAa,CAAC;AAUnE,MAAM,WAAW,GAAG,EAAE,GAAG,IAAI,CAAC;AAC9B,MAAM,aAAa,GAAG,SAAS,CAAC;AAChC,SAAS,qBAAqB;IAC5B,SAAS,CACP,SAAS,CAAC,UAAU,IAAI,SAAS,CAAC,UAAU,EAC5C,0DAA0D,CAC3D,CAAC;AACJ,CAAC;AACD,MAAM,UAAU,GAAG,CAAC,SAAiB,EAAE,MAAc,EAAE,EAAE;IACvD,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,iCAAiC,CAAC,CAAC;IAC5E,OAAO,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,GAAG,MAAM,QAAQ,CAAC,CAAC;AACvD,CAAC,CAAC;AAEF,MAAM,UAAU,eAAe,CAAC,CAAQ;IACtC,SAAS,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,8BAA8B,CAAC,CAAC;IAC5E,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAC3B,MAAM,QAAQ,GAAG,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC;IACpE,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1D,SAAS,CACP,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,MAAM;QAC5B,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC;QAC5B,CAAC,CAAC,WAAW,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,EAC1C,uCAAuC,CACxC,CAAC;IACF,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACtC,SAAS,CACP,IAAI;YACF,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ;YAC/B,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YACpC,KAAK;YACL,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC;YAC7B,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC;YACtB,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ;YAC/B,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YAClC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC;YACrC,IAAI,CAAC,UAAU,GAAG,CAAC;YACnB,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC;YACtC,IAAI,CAAC,WAAW,KAAK,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAChD,kCAAkC,CACnC,CAAC;QACF,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACxB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,SAAiB;IAC9C,qBAAqB,EAAE,CAAC;IACxB,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,SAAS,CACP,IAAI,CAAC,WAAW,EAAE;QAChB,CAAC,IAAI,CAAC,cAAc,EAAE;QACtB,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,MAAM,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,CAAC,EACvE,qDAAqD,CACtD,CAAC;AACJ,CAAC;AACD,KAAK,UAAU,UAAU,CACvB,SAAiB,EACjB,IAAgB;IAEhB,qBAAqB,EAAE,CAAC;IACxB,MAAM,IAAI,GAAG,MAAM,IAAI,CACrB,UAAU,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,EAClC,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,UAAU,GAAG,SAAS,CAAC,UAAU,CACjE,CAAC;IACF,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAC/B,SAAS,CACP,IAAI,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,UAAU,EAC9C,6DAA6D,CAC9D,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,sGAAsG;AACtG,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,eAAe,CAAC,CAAQ;IAC7C,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnB,MAAM,KAAK,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE;YACjE,QAAQ,EAAE,MAAM;SACjB,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC9D,IAAI,CAAC;YACH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,KAAK;gBAAE,IAAI,IAAI;oBAAE,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAQ,CAAC;QAC1E,CAAC;gBAAS,CAAC;YACT,KAAK,CAAC,KAAK,EAAE,CAAC;YACd,KAAK,CAAC,OAAO,EAAE,CAAC;QAClB,CAAC;QACD,OAAO;IACT,CAAC;IACD,eAAe,CAAC,CAAC,CAAC,CAAC;IACnB,MAAM,eAAe,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACnC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,WAAY,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,CAAC,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC;YAClC,SAAS,EAAE,KAAK;YAChB,aAAa,EAAE,WAAW;SAC3B,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,EACjC,OAAO,GAAG,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,IAAI,OAAO,GAAG,EAAE,EACd,KAAK,GAAG,CAAC,EACT,KAAK,GAAG,CAAC,CAAC;QACZ,IAAI,CAAC;YACH,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;gBAChC,MAAM,IAAI,GAAG,KAAe,CAAC;gBAC7B,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACpB,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC;gBACrB,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;gBAClD,IAAI,OAAe,CAAC;gBACpB,OAAO,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC9C,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;oBACvC,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;oBACrC,SAAS,CACP,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,aAAa,EACvD,+BAA+B,CAChC,CAAC;oBACF,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAQ,CAAC;oBACpC,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAE,CAAC;oBACtC,SAAS,CACP,KAAK,KAAK,CAAC;wBACT,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,MAAM,IAAI,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,MAAM;wBAC/C,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,OAAO;4BAClB,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM;4BAC1B,GAAG,CAAC,EAAE,KAAK,KAAK,CAAC,QAAS,CAAC,KAAK,GAAG,CAAC,CAAC,EAC3C,oCAAoC,CACrC,CAAC;oBACF,KAAK,EAAE,CAAC;oBACR,MAAM,GAAG,CAAC;gBACZ,CAAC;gBACD,SAAS,CACP,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,aAAa,EAC3C,8CAA8C,CAC/C,CAAC;YACJ,CAAC;YACD,OAAO,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YAC5B,SAAS,CACP,CAAC,OAAO;gBACN,KAAK,KAAK,IAAI,CAAC,UAAU;gBACzB,KAAK,KAAK,IAAI,CAAC,WAAW;gBAC1B,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,MAAM,EACtC,wCAAwC,CACzC,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,KAAK,CAAC,OAAO,EAAE,CAAC;YAChB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;AACH,CAAC;AAED,mGAAmG;AACnG,MAAM,OAAO,cAAc;IAcd;IACA;IAdF,YAAY,GAA2B,EAAE,CAAC;IAC1C,KAAK,GAAiB,EAAE,CAAC;IAC1B,MAAM,CAAc;IACpB,OAAO,GAAa,EAAE,CAAC;IACvB,YAAY,GAAG,CAAC,CAAC;IACjB,IAAI,CAMV;IACF,YACW,SAAiB,EACjB,OAAwB;QADxB,cAAS,GAAT,SAAS,CAAQ;QACjB,YAAO,GAAP,OAAO,CAAiB;IAChC,CAAC;IACJ,MAAM,CAAC,KAAK,CAAC,MAAM,CACjB,SAAiB,EACjB,OAAwB;QAExB,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACtD,IAAI,OAAO,KAAK,OAAO;YACrB,MAAM,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;aACvE,CAAC;YACJ,qBAAqB,EAAE,CAAC;YACxB,MAAM,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3D,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IACO,KAAK,CAAC,KAAK;QACjB,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,OAAO;QACjC,MAAM,IAAI,CAAC,MAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAChD,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAClB,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;IACxB,CAAC;IACO,KAAK,CAAC,UAAU;QACtB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,IAAI,CAAC,IAAI;YAAE,OAAO;QACvB,MAAM,IAAI,CAAC,MAAO,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACzC,+EAA+E;QAC/E,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;QAC5D,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;YACd,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM;YACN,UAAU,EAAE,IAAI,CAAC,KAAK;YACtB,WAAW,EAAE,IAAI,CAAC,KAAK;SACxB,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,GAAG,SAAS,CAAC;IACxB,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,GAAQ;QACjB,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,EACzB,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAClC,SAAS,CACP,KAAK,IAAI,aAAa,EACtB,2CAA2C,GAAG,CAAC,EAAE,EAAE,CACpD,CAAC;QACF,SAAS,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,wBAAwB,CAAC,CAAC;QAChE,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,IAAI,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;YAChC,IAAI,GAAG,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACxB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;gBACxB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,GAAG,UAAU,EAAE,MAAM,CAAC,CAAC;gBACpE,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;gBAC5C,IAAI,CAAC,IAAI,GAAG;oBACV,MAAM,EAAE,GAAG,CAAC,EAAE;oBACd,IAAI;oBACJ,MAAM,EAAE,UAAU,CAAC,QAAQ,CAAC;oBAC5B,KAAK,EAAE,CAAC;oBACR,KAAK,EAAE,CAAC;iBACT,CAAC;YACJ,CAAC;YACD,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,wBAAwB,CAAC,CAAC;YAC/C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;YACrC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,KAAK,GAAG,CAAC,CAAC;YAC7B,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QACpB,CAAC;QACD,IAAI,IAAI,CAAC,YAAY,GAAG,KAAK,GAAG,CAAC,GAAG,WAAW;YAAE,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACpE,IAAI,KAAK,GAAG,CAAC,IAAI,WAAW;YAAE,MAAM,IAAI,CAAC,MAAO,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;aAC/D,CAAC;YACJ,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;YAC/B,IAAI,CAAC,YAAY,IAAI,KAAK,GAAG,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IACD,KAAK,CAAC,MAAM,CACV,QAAe,EACf,IAAgB,EAChB,GAAsB;QAEtB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACxB,MAAM,eAAe,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QACxD,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,EACxD,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACnD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC7B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IACE,CAAC,CAAC,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC,QAAQ,CACnD,KAA+B,CAAC,IAAI,IAAI,EAAE,CAC5C;gBAED,MAAM,KAAK,CAAC;YACd,MAAM,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,CAAC,aAAa,CAAC,CAAC;QAC1D,CAAC;QACD,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;YACrB,SAAS,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,EAAE,wBAAwB,CAAC,CAAC;YAC5D,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAE,CAAC;QACrD,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;IAC/B,CAAC;IACD,KAAK,CAAC,MAAM;QACV,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;IAC1B,CAAC;IACD,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;IAC1B,CAAC;CACF"}
|
package/dist/build.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { resolveCommit, type Identity } from "./git.js";
|
|
2
2
|
import { CHUNKER } from "./chunk.js";
|
|
3
|
+
import { type ArtifactStorage, type RecordPart } from "./artifact.js";
|
|
3
4
|
import { type Doc } from "./common.js";
|
|
4
5
|
import { type FilePolicy } from "./file-policy.js";
|
|
5
6
|
export declare const ANALYZERS: readonly ["english", "japanese", "korean", "standard"];
|
|
@@ -130,7 +131,8 @@ export type InventoryItem = {
|
|
|
130
131
|
skipped?: number;
|
|
131
132
|
};
|
|
132
133
|
export type Build = {
|
|
133
|
-
format: 1;
|
|
134
|
+
format: 1 | 2;
|
|
135
|
+
recordParts?: RecordPart[];
|
|
134
136
|
directory: string;
|
|
135
137
|
repoKey: string;
|
|
136
138
|
configHash: string;
|
|
@@ -158,7 +160,7 @@ export type Embedder = {
|
|
|
158
160
|
dimensions: number;
|
|
159
161
|
embed: (text: string) => Promise<number[]>;
|
|
160
162
|
};
|
|
161
|
-
export declare function records(
|
|
163
|
+
export declare function records(build: Build | string): AsyncGenerator<Doc>;
|
|
162
164
|
export declare function loadBuild(directory: string): Promise<Build>;
|
|
163
165
|
export declare function materialize(args: {
|
|
164
166
|
identity: Identity;
|
|
@@ -166,6 +168,7 @@ export declare function materialize(args: {
|
|
|
166
168
|
resolvedCommit?: Awaited<ReturnType<typeof resolveCommit>>;
|
|
167
169
|
output: string;
|
|
168
170
|
previous?: Build;
|
|
171
|
+
storage?: ArtifactStorage;
|
|
169
172
|
preset?: Preset;
|
|
170
173
|
embedder?: Embedder;
|
|
171
174
|
cache?: string;
|