@semiont/content 0.5.27 → 0.5.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/index.d.ts +108 -57
- package/dist/index.js +153 -84
- package/dist/index.js.map +1 -1
- package/package.json +6 -6
package/README.md
CHANGED
|
@@ -16,7 +16,7 @@ npm install @semiont/content
|
|
|
16
16
|
|
|
17
17
|
## Architecture Context
|
|
18
18
|
|
|
19
|
-
**Infrastructure Ownership**: In production applications, the working tree store is **created and managed by [@semiont/make-meaning](../make-meaning/)'s `startMakeMeaning()` function**, which serves as the single orchestration point for all infrastructure components.
|
|
19
|
+
**Infrastructure Ownership**: In production applications, the working tree store is **created and managed by [@semiont/make-meaning](../make-meaning/)'s `startMakeMeaning()` function**, which serves as the single orchestration point for all infrastructure components. Gateway code accesses it as `knowledgeBase.content`.
|
|
20
20
|
|
|
21
21
|
The quick start example below shows direct instantiation for **testing, CLI tools, or content management scripts**.
|
|
22
22
|
|
|
@@ -26,7 +26,9 @@ The quick start example below shows direct instantiation for **testing, CLI tool
|
|
|
26
26
|
import { WorkingTreeStore, deriveStorageUri } from '@semiont/content';
|
|
27
27
|
import { SemiontProject } from '@semiont/core/node';
|
|
28
28
|
|
|
29
|
-
const project = new SemiontProject('/path/to/project'
|
|
29
|
+
const project = new SemiontProject('/path/to/project', {
|
|
30
|
+
anchoredTextDir: process.env.SEMIONT_ANCHORED_TEXT_DIR!,
|
|
31
|
+
});
|
|
30
32
|
const store = new WorkingTreeStore(project);
|
|
31
33
|
|
|
32
34
|
// Derive a stable file:// URI from a resource name
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { Readable } from 'stream';
|
|
2
|
+
import { SemiontProject, ArchivistAddressConfig } from '@semiont/core/node';
|
|
3
|
+
import { Logger, ExtractionOutcome, PdfTextItem, TextExtraction, IContentTransport, AnchoredText } from '@semiont/core';
|
|
3
4
|
|
|
4
5
|
/**
|
|
5
6
|
* WorkingTreeStore - Manages files in the project working tree
|
|
@@ -12,9 +13,11 @@ import { Logger, SupportedMediaType, ExtractionOutcome, PdfTextItem, TextExtract
|
|
|
12
13
|
* Two write paths:
|
|
13
14
|
* - store(content, storageUri): Write bytes to disk (API/GUI/AI path).
|
|
14
15
|
* Used when the file does not yet exist and the caller provides content.
|
|
15
|
-
* - register(storageUri, expectedChecksum?):
|
|
16
|
-
* return its metadata
|
|
17
|
-
*
|
|
16
|
+
* - register(storageUri, expectedChecksum?): Adopt a file already on disk and
|
|
17
|
+
* return its metadata. The CLI path (the file arrived by other means) and
|
|
18
|
+
* the event-apply path (the Stower staging bytes an event names) both use
|
|
19
|
+
* it. Streams the file to hash it — never holds it. If expectedChecksum is
|
|
20
|
+
* provided, throws on mismatch.
|
|
18
21
|
*
|
|
19
22
|
* Storage layout:
|
|
20
23
|
* {projectRoot}/{path-from-uri}
|
|
@@ -44,20 +47,33 @@ declare class WorkingTreeStore {
|
|
|
44
47
|
/**
|
|
45
48
|
* Write content to disk at the location indicated by storageUri.
|
|
46
49
|
*
|
|
47
|
-
* API/GUI/AI path: caller provides bytes
|
|
50
|
+
* API/GUI/AI path: caller provides bytes — as a Buffer it already holds, or
|
|
51
|
+
* as a stream (the Archivist's write endpoint hands the request body
|
|
52
|
+
* straight through, SINGLE-KB-MOUNT P2/D7: memory stays bounded by the
|
|
53
|
+
* chunk, never the representation).
|
|
48
54
|
*
|
|
49
|
-
*
|
|
55
|
+
* Atomic either way: bytes stream into a temp file beside the target and
|
|
56
|
+
* are renamed into place only once complete — and only once
|
|
57
|
+
* `expectedChecksum`, when given, agrees with what actually arrived. A
|
|
58
|
+
* mismatch or a torn stream leaves the target untouched (a version being
|
|
59
|
+
* overwritten survives) and no temp file behind, so the Stower's `register`
|
|
60
|
+
* can never find partial bytes an event names.
|
|
61
|
+
*
|
|
62
|
+
* @param content - Raw bytes to write, whole or streamed
|
|
50
63
|
* @param storageUri - file:// URI (e.g. "file://docs/overview.md")
|
|
64
|
+
* @throws ChecksumMismatchError when expectedChecksum disagrees with the body
|
|
51
65
|
* @returns Stored resource metadata
|
|
52
66
|
*/
|
|
53
|
-
store(content: Buffer, storageUri: string, options?: {
|
|
67
|
+
store(content: Buffer | Readable, storageUri: string, options?: {
|
|
54
68
|
noGit?: boolean;
|
|
69
|
+
expectedChecksum?: string;
|
|
55
70
|
}): Promise<StoredResource>;
|
|
56
71
|
/**
|
|
57
72
|
* Read an existing file and return its metadata.
|
|
58
73
|
*
|
|
59
|
-
*
|
|
60
|
-
* If expectedChecksum is provided, throws
|
|
74
|
+
* The file is already on disk; this hashes it by streaming to confirm what
|
|
75
|
+
* it is, then stages it. If expectedChecksum is provided, throws
|
|
76
|
+
* ChecksumMismatchError on mismatch.
|
|
61
77
|
*
|
|
62
78
|
* @param storageUri - file:// URI (e.g. "file://docs/overview.md")
|
|
63
79
|
* @param expectedChecksum - Optional SHA-256 to verify against
|
|
@@ -74,6 +90,19 @@ declare class WorkingTreeStore {
|
|
|
74
90
|
* @param storageUri - file:// URI
|
|
75
91
|
* @returns Raw bytes
|
|
76
92
|
*/
|
|
93
|
+
/**
|
|
94
|
+
* The same bytes as `retrieve`, streamed — for the byte paths that must not
|
|
95
|
+
* hold a whole representation in memory (SINGLE-KB-MOUNT D7: the Archivist
|
|
96
|
+
* serves content for every reader now, so its memory cannot be bounded by
|
|
97
|
+
* the largest file anyone asks for).
|
|
98
|
+
*
|
|
99
|
+
* Lazy by construction: the stream is created here but nothing is read
|
|
100
|
+
* until the caller iterates, so a missing file surfaces as an `error` event
|
|
101
|
+
* on the stream rather than a rejected promise. Callers that need the
|
|
102
|
+
* distinction up front should resolve the descriptor first — which is what
|
|
103
|
+
* `resolveRepresentation` does.
|
|
104
|
+
*/
|
|
105
|
+
retrieveStream(storageUri: string): Readable;
|
|
77
106
|
retrieve(storageUri: string): Promise<Buffer>;
|
|
78
107
|
/**
|
|
79
108
|
* Move a file from one URI to another.
|
|
@@ -128,26 +157,6 @@ declare class ChecksumMismatchError extends Error {
|
|
|
128
157
|
constructor(storageUri: string, expected: string, actual: string);
|
|
129
158
|
}
|
|
130
159
|
|
|
131
|
-
/**
|
|
132
|
-
* Storage URI Derivation
|
|
133
|
-
*
|
|
134
|
-
* Builds the file:// URI a resource lives at in the working tree from its
|
|
135
|
-
* name and validated media type. Extensions come from the media-type
|
|
136
|
-
* registry in @semiont/core; formats are validated upstream at the
|
|
137
|
-
* create/yield boundary, so the lookup is strict — no fallback.
|
|
138
|
-
*/
|
|
139
|
-
|
|
140
|
-
/**
|
|
141
|
-
* Derive a file:// storage URI from a resource name and media type.
|
|
142
|
-
*
|
|
143
|
-
* The name is lowercased, runs of non-alphanumeric characters collapse to
|
|
144
|
-
* single hyphens, and leading/trailing hyphens are stripped.
|
|
145
|
-
*
|
|
146
|
-
* @example
|
|
147
|
-
* deriveStorageUri("My Document", "text/markdown") // => "file://my-document.md"
|
|
148
|
-
*/
|
|
149
|
-
declare function deriveStorageUri(name: string, format: SupportedMediaType): string;
|
|
150
|
-
|
|
151
160
|
/**
|
|
152
161
|
* Checksum utilities for content verification
|
|
153
162
|
*/
|
|
@@ -187,13 +196,6 @@ declare function verifyChecksum(content: string | Buffer, checksum: string): boo
|
|
|
187
196
|
* "no entry under the current checksum" as work (P0's third drift class).
|
|
188
197
|
*/
|
|
189
198
|
|
|
190
|
-
/** The two halves of the wire record, split for storage. */
|
|
191
|
-
type SuccessOutcome = Exclude<ExtractionOutcome, {
|
|
192
|
-
declined: string;
|
|
193
|
-
}>;
|
|
194
|
-
type DeclineOutcome = Extract<ExtractionOutcome, {
|
|
195
|
-
declined: string;
|
|
196
|
-
}>;
|
|
197
199
|
/**
|
|
198
200
|
* One line of recognized text: the geometry every word on it shares, plus the
|
|
199
201
|
* per-word parts that differ.
|
|
@@ -244,10 +246,14 @@ type CachedAnchoredText = ({
|
|
|
244
246
|
stamp: string;
|
|
245
247
|
text: string;
|
|
246
248
|
lines: CachedLine[];
|
|
247
|
-
} & Omit<
|
|
249
|
+
} & Omit<Extract<ExtractionOutcome, {
|
|
250
|
+
kind: 'extracted';
|
|
251
|
+
}>, 'kind' | 'text' | 'items'>) | ({
|
|
248
252
|
v: 2;
|
|
249
253
|
stamp: string;
|
|
250
|
-
} &
|
|
254
|
+
} & Omit<Extract<ExtractionOutcome, {
|
|
255
|
+
kind: 'declined';
|
|
256
|
+
}>, 'kind'>);
|
|
251
257
|
interface AnchoredTextStore {
|
|
252
258
|
/**
|
|
253
259
|
* The stored map for this key, or null for any miss. Never throws.
|
|
@@ -261,8 +267,19 @@ interface AnchoredTextStore {
|
|
|
261
267
|
* key scheme here.
|
|
262
268
|
*/
|
|
263
269
|
read(key: string): Promise<ExtractionOutcome | null>;
|
|
264
|
-
/**
|
|
265
|
-
*
|
|
270
|
+
/**
|
|
271
|
+
* Record an extraction outcome under the content checksum of its source
|
|
272
|
+
* bytes. **THROWS on failure: a write that returns has written.**
|
|
273
|
+
*
|
|
274
|
+
* Asymmetric with `read` above, which never throws, and deliberately so —
|
|
275
|
+
* a miss is a normal answer, a failed write is not. The store used to
|
|
276
|
+
* swallow for everyone, which forced the one caller that needs a throw
|
|
277
|
+
* (the Smelter's re-anchor publish, whose `smelt:rebuild-anchors-failed`
|
|
278
|
+
* accounting rides on it) to route around the store entirely. Now the
|
|
279
|
+
* contract is honest and **leniency is the caller's**, stated where it is
|
|
280
|
+
* wanted: the read-through seam in `pdf-extractor` catches, because a
|
|
281
|
+
* store may make extraction faster but must never make it fail.
|
|
282
|
+
*/
|
|
266
283
|
write(key: string, outcome: ExtractionOutcome): Promise<void>;
|
|
267
284
|
/**
|
|
268
285
|
* Every key `read()` would currently HIT — entries under a stale stamp or
|
|
@@ -307,6 +324,8 @@ declare function createAnchoredTextStore(dir: string, logger?: Logger): Anchored
|
|
|
307
324
|
*/
|
|
308
325
|
|
|
309
326
|
interface ExtractedText {
|
|
327
|
+
/** Discriminant — mirrors the wire member (WIRE-UNION-DISCRIMINANTS P5c/D6). */
|
|
328
|
+
kind: 'extracted';
|
|
310
329
|
/** Reading-order plain text, ready for the chunker. */
|
|
311
330
|
text: string;
|
|
312
331
|
/**
|
|
@@ -351,6 +370,8 @@ interface ExtractedText {
|
|
|
351
370
|
* could not name its class; SMELTER-MEDIA-TYPES Phase 0 log, note a).
|
|
352
371
|
*/
|
|
353
372
|
interface ExtractionDecline {
|
|
373
|
+
/** Discriminant — mirrors the wire member (WIRE-UNION-DISCRIMINANTS P5c/D6). */
|
|
374
|
+
kind: 'declined';
|
|
354
375
|
declined: 'no-text-layer' | 'encrypted' | 'corrupt' | 'too-large';
|
|
355
376
|
}
|
|
356
377
|
/**
|
|
@@ -439,24 +460,54 @@ declare const MAX_PDF_BYTES: number;
|
|
|
439
460
|
declare function withinByteBudget(bytes: number): boolean;
|
|
440
461
|
|
|
441
462
|
/**
|
|
442
|
-
*
|
|
443
|
-
*
|
|
463
|
+
* Reading a resource's bytes: the contract, the way it fails, and the
|
|
464
|
+
* implementation that reaches the Archivist over HTTP.
|
|
465
|
+
*
|
|
466
|
+
* These live in `@semiont/content` because this package IS the byte layer —
|
|
467
|
+
* the Archivist's whole job — and because the readers span the dependency
|
|
468
|
+
* graph. `@semiont/make-meaning` holds the Archivist itself and satisfies
|
|
469
|
+
* `ContentReads` in-process from the working tree; `@semiont/jobs` holds the
|
|
470
|
+
* Worker and can only reach the record over the wire. make-meaning depends on
|
|
471
|
+
* jobs, so anything both need has to sit under both (SINGLE-KB-MOUNT P4).
|
|
444
472
|
*
|
|
445
|
-
*
|
|
446
|
-
*
|
|
447
|
-
*
|
|
448
|
-
*
|
|
449
|
-
*
|
|
450
|
-
* over the wire (HttpContentTransport → the /anchored-text routes).
|
|
473
|
+
* Where the Archivist IS lives in `@semiont/core/node` (`archivistEndpoint`),
|
|
474
|
+
* not here: an address is a config value plus an environment variable, and
|
|
475
|
+
* the gateway needs it without needing a byte reader. One resolution, shared
|
|
476
|
+
* with the gateway's own proxying — the address and the secret are deployment
|
|
477
|
+
* facts, and a second copy of either is a second thing to get wrong.
|
|
451
478
|
*
|
|
452
|
-
*
|
|
453
|
-
*
|
|
454
|
-
*
|
|
455
|
-
* — the re-anchor path, whose artifact IS the job — use the transport's
|
|
456
|
-
* `putAnchoredText` directly, not this adapter.
|
|
479
|
+
* Absence fails loudly. A missing host or secret is a misconfiguration, never
|
|
480
|
+
* a reason to fall back to reading a tree locally — the point of
|
|
481
|
+
* SINGLE-KB-MOUNT is that exactly one process touches it.
|
|
457
482
|
*/
|
|
458
483
|
|
|
459
|
-
|
|
484
|
+
/**
|
|
485
|
+
* The byte read, and nothing else — DERIVED from the transport contract so it
|
|
486
|
+
* cannot drift from it. Keyed by ResourceId because that is the transport's
|
|
487
|
+
* key and the Archivist's: no caller converts to a tree address only to have
|
|
488
|
+
* the far side convert back.
|
|
489
|
+
*/
|
|
490
|
+
type ContentReads = Pick<IContentTransport, 'getBinary'>;
|
|
491
|
+
/** Which half of the lookup failed — the gateway serves two different 404s. */
|
|
492
|
+
type MissingReason = 'resource' | 'representation';
|
|
493
|
+
declare class RepresentationMissing extends Error {
|
|
494
|
+
readonly resourceId: string;
|
|
495
|
+
readonly reason: MissingReason;
|
|
496
|
+
constructor(resourceId: string, reason: MissingReason);
|
|
497
|
+
}
|
|
498
|
+
/**
|
|
499
|
+
* `ContentReads` against the Archivist — how a fleet process that holds no KB
|
|
500
|
+
* mount reads bytes (SINGLE-KB-MOUNT P4).
|
|
501
|
+
*
|
|
502
|
+
* The address resolves HERE, at construction, not per read: a process with no
|
|
503
|
+
* Archivist configured must die while an operator is watching it boot, rather
|
|
504
|
+
* than fail every resource for the life of the process.
|
|
505
|
+
*
|
|
506
|
+
* A miss arrives as `RepresentationMissing` — the same error the in-process
|
|
507
|
+
* face throws for the same fact, so no caller can tell whether the bytes were
|
|
508
|
+
* a hop away. `reason` rides the wire precisely so this side need not guess.
|
|
509
|
+
*/
|
|
510
|
+
declare function archivistContentReads(config: ArchivistAddressConfig): ContentReads;
|
|
460
511
|
|
|
461
512
|
/**
|
|
462
513
|
* PDF Text Layer Types
|
|
@@ -532,5 +583,5 @@ interface PdfTextLayer extends AnchoredText {
|
|
|
532
583
|
|
|
533
584
|
declare function extractPdfTextLayer(bytes: Uint8Array | Buffer): Promise<PdfTextLayer | null>;
|
|
534
585
|
|
|
535
|
-
export { ChecksumMismatchError, EXTRACTORS, MAX_PDF_BYTES, WorkingTreeStore,
|
|
536
|
-
export type { AnchoredTextStore, CachedAnchoredText, CachedLine, ContentExtractor, ExtractedText, ExtractionCache, ExtractionDecline, PdfFormField, PdfPageInfo, PdfTextLayer, StoredResource };
|
|
586
|
+
export { ChecksumMismatchError, EXTRACTORS, MAX_PDF_BYTES, RepresentationMissing, WorkingTreeStore, archivistContentReads, calculateChecksum, createAnchoredTextStore, extractPdfTextLayer, verifyChecksum, withinByteBudget };
|
|
587
|
+
export type { AnchoredTextStore, CachedAnchoredText, CachedLine, ContentExtractor, ContentReads, ExtractedText, ExtractionCache, ExtractionDecline, MissingReason, PdfFormField, PdfPageInfo, PdfTextLayer, StoredResource };
|
package/dist/index.js
CHANGED
|
@@ -1,20 +1,26 @@
|
|
|
1
1
|
// src/working-tree-store.ts
|
|
2
|
-
import { promises as fs } from "fs";
|
|
2
|
+
import { promises as fs, createReadStream, createWriteStream } from "fs";
|
|
3
3
|
import { execFileSync } from "child_process";
|
|
4
|
+
import { createHash, randomUUID } from "crypto";
|
|
5
|
+
import { Readable } from "stream";
|
|
6
|
+
import { pipeline } from "stream/promises";
|
|
4
7
|
import path from "path";
|
|
5
|
-
|
|
6
|
-
// src/checksum.ts
|
|
7
|
-
import { createHash } from "crypto";
|
|
8
|
-
function calculateChecksum(content) {
|
|
8
|
+
function hashingTap() {
|
|
9
9
|
const hash = createHash("sha256");
|
|
10
|
-
|
|
11
|
-
return
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
10
|
+
let byteSize = 0;
|
|
11
|
+
return {
|
|
12
|
+
update(chunk2) {
|
|
13
|
+
hash.update(chunk2);
|
|
14
|
+
byteSize += chunk2.length;
|
|
15
|
+
},
|
|
16
|
+
get byteSize() {
|
|
17
|
+
return byteSize;
|
|
18
|
+
},
|
|
19
|
+
digest() {
|
|
20
|
+
return hash.digest("hex");
|
|
21
|
+
}
|
|
22
|
+
};
|
|
15
23
|
}
|
|
16
|
-
|
|
17
|
-
// src/working-tree-store.ts
|
|
18
24
|
var WorkingTreeStore = class {
|
|
19
25
|
projectRoot;
|
|
20
26
|
gitSync;
|
|
@@ -30,34 +36,68 @@ var WorkingTreeStore = class {
|
|
|
30
36
|
/**
|
|
31
37
|
* Write content to disk at the location indicated by storageUri.
|
|
32
38
|
*
|
|
33
|
-
* API/GUI/AI path: caller provides bytes
|
|
39
|
+
* API/GUI/AI path: caller provides bytes — as a Buffer it already holds, or
|
|
40
|
+
* as a stream (the Archivist's write endpoint hands the request body
|
|
41
|
+
* straight through, SINGLE-KB-MOUNT P2/D7: memory stays bounded by the
|
|
42
|
+
* chunk, never the representation).
|
|
34
43
|
*
|
|
35
|
-
*
|
|
44
|
+
* Atomic either way: bytes stream into a temp file beside the target and
|
|
45
|
+
* are renamed into place only once complete — and only once
|
|
46
|
+
* `expectedChecksum`, when given, agrees with what actually arrived. A
|
|
47
|
+
* mismatch or a torn stream leaves the target untouched (a version being
|
|
48
|
+
* overwritten survives) and no temp file behind, so the Stower's `register`
|
|
49
|
+
* can never find partial bytes an event names.
|
|
50
|
+
*
|
|
51
|
+
* @param content - Raw bytes to write, whole or streamed
|
|
36
52
|
* @param storageUri - file:// URI (e.g. "file://docs/overview.md")
|
|
53
|
+
* @throws ChecksumMismatchError when expectedChecksum disagrees with the body
|
|
37
54
|
* @returns Stored resource metadata
|
|
38
55
|
*/
|
|
39
56
|
async store(content, storageUri, options) {
|
|
40
57
|
const filePath = this.resolveUri(storageUri);
|
|
41
|
-
const
|
|
42
|
-
this.logger?.debug("Storing resource", { storageUri
|
|
58
|
+
const source = Buffer.isBuffer(content) ? Readable.from([content]) : content;
|
|
59
|
+
this.logger?.debug("Storing resource", { storageUri });
|
|
43
60
|
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
61
|
+
const tempPath = `${filePath}.${randomUUID()}.tmp`;
|
|
62
|
+
const tap = hashingTap();
|
|
63
|
+
try {
|
|
64
|
+
await pipeline(
|
|
65
|
+
source,
|
|
66
|
+
async function* (chunks) {
|
|
67
|
+
for await (const chunk2 of chunks) {
|
|
68
|
+
tap.update(chunk2);
|
|
69
|
+
yield chunk2;
|
|
70
|
+
}
|
|
71
|
+
},
|
|
72
|
+
createWriteStream(tempPath)
|
|
73
|
+
);
|
|
74
|
+
const checksum = tap.digest();
|
|
75
|
+
const byteSize = tap.byteSize;
|
|
76
|
+
if (options?.expectedChecksum !== void 0 && options.expectedChecksum !== checksum) {
|
|
77
|
+
throw new ChecksumMismatchError(storageUri, options.expectedChecksum, checksum);
|
|
78
|
+
}
|
|
79
|
+
await fs.rename(tempPath, filePath);
|
|
80
|
+
if (this.shouldRunGit(options?.noGit)) {
|
|
81
|
+
execFileSync("git", ["add", filePath], { cwd: this.projectRoot });
|
|
82
|
+
}
|
|
83
|
+
this.logger?.info("Resource stored", { storageUri, checksum, byteSize });
|
|
84
|
+
return {
|
|
85
|
+
storageUri,
|
|
86
|
+
checksum,
|
|
87
|
+
byteSize,
|
|
88
|
+
created: (/* @__PURE__ */ new Date()).toISOString()
|
|
89
|
+
};
|
|
90
|
+
} catch (error) {
|
|
91
|
+
await fs.rm(tempPath, { force: true });
|
|
92
|
+
throw error;
|
|
47
93
|
}
|
|
48
|
-
this.logger?.info("Resource stored", { storageUri, checksum, byteSize: content.length });
|
|
49
|
-
return {
|
|
50
|
-
storageUri,
|
|
51
|
-
checksum,
|
|
52
|
-
byteSize: content.length,
|
|
53
|
-
created: (/* @__PURE__ */ new Date()).toISOString()
|
|
54
|
-
};
|
|
55
94
|
}
|
|
56
95
|
/**
|
|
57
96
|
* Read an existing file and return its metadata.
|
|
58
97
|
*
|
|
59
|
-
*
|
|
60
|
-
* If expectedChecksum is provided, throws
|
|
98
|
+
* The file is already on disk; this hashes it by streaming to confirm what
|
|
99
|
+
* it is, then stages it. If expectedChecksum is provided, throws
|
|
100
|
+
* ChecksumMismatchError on mismatch.
|
|
61
101
|
*
|
|
62
102
|
* @param storageUri - file:// URI (e.g. "file://docs/overview.md")
|
|
63
103
|
* @param expectedChecksum - Optional SHA-256 to verify against
|
|
@@ -68,19 +108,23 @@ var WorkingTreeStore = class {
|
|
|
68
108
|
async register(storageUri, expectedChecksum, options) {
|
|
69
109
|
const filePath = this.resolveUri(storageUri);
|
|
70
110
|
this.logger?.debug("Registering resource", { storageUri });
|
|
71
|
-
const
|
|
72
|
-
const
|
|
73
|
-
|
|
111
|
+
const tap = hashingTap();
|
|
112
|
+
for await (const chunk2 of createReadStream(filePath)) {
|
|
113
|
+
tap.update(chunk2);
|
|
114
|
+
}
|
|
115
|
+
const checksum = tap.digest();
|
|
116
|
+
if (expectedChecksum !== void 0 && checksum !== expectedChecksum) {
|
|
74
117
|
throw new ChecksumMismatchError(storageUri, expectedChecksum, checksum);
|
|
75
118
|
}
|
|
76
119
|
if (this.shouldRunGit(options?.noGit)) {
|
|
77
120
|
execFileSync("git", ["add", filePath], { cwd: this.projectRoot });
|
|
78
121
|
}
|
|
79
|
-
|
|
122
|
+
const byteSize = tap.byteSize;
|
|
123
|
+
this.logger?.info("Resource registered", { storageUri, checksum, byteSize });
|
|
80
124
|
return {
|
|
81
125
|
storageUri,
|
|
82
126
|
checksum,
|
|
83
|
-
byteSize
|
|
127
|
+
byteSize,
|
|
84
128
|
created: (/* @__PURE__ */ new Date()).toISOString()
|
|
85
129
|
};
|
|
86
130
|
}
|
|
@@ -90,6 +134,21 @@ var WorkingTreeStore = class {
|
|
|
90
134
|
* @param storageUri - file:// URI
|
|
91
135
|
* @returns Raw bytes
|
|
92
136
|
*/
|
|
137
|
+
/**
|
|
138
|
+
* The same bytes as `retrieve`, streamed — for the byte paths that must not
|
|
139
|
+
* hold a whole representation in memory (SINGLE-KB-MOUNT D7: the Archivist
|
|
140
|
+
* serves content for every reader now, so its memory cannot be bounded by
|
|
141
|
+
* the largest file anyone asks for).
|
|
142
|
+
*
|
|
143
|
+
* Lazy by construction: the stream is created here but nothing is read
|
|
144
|
+
* until the caller iterates, so a missing file surfaces as an `error` event
|
|
145
|
+
* on the stream rather than a rejected promise. Callers that need the
|
|
146
|
+
* distinction up front should resolve the descriptor first — which is what
|
|
147
|
+
* `resolveRepresentation` does.
|
|
148
|
+
*/
|
|
149
|
+
retrieveStream(storageUri) {
|
|
150
|
+
return createReadStream(this.resolveUri(storageUri));
|
|
151
|
+
}
|
|
93
152
|
async retrieve(storageUri) {
|
|
94
153
|
const filePath = this.resolveUri(storageUri);
|
|
95
154
|
try {
|
|
@@ -195,11 +254,15 @@ The file on disk differs from the recorded checksum. Has it been modified since
|
|
|
195
254
|
actual;
|
|
196
255
|
};
|
|
197
256
|
|
|
198
|
-
// src/
|
|
199
|
-
import {
|
|
200
|
-
function
|
|
201
|
-
const
|
|
202
|
-
|
|
257
|
+
// src/checksum.ts
|
|
258
|
+
import { createHash as createHash2 } from "crypto";
|
|
259
|
+
function calculateChecksum(content) {
|
|
260
|
+
const hash = createHash2("sha256");
|
|
261
|
+
hash.update(content);
|
|
262
|
+
return hash.digest("hex");
|
|
263
|
+
}
|
|
264
|
+
function verifyChecksum(content, checksum) {
|
|
265
|
+
return calculateChecksum(content) === checksum;
|
|
203
266
|
}
|
|
204
267
|
|
|
205
268
|
// src/content-extractor.ts
|
|
@@ -725,7 +788,7 @@ function foldFormFields(layer) {
|
|
|
725
788
|
height: field.height
|
|
726
789
|
});
|
|
727
790
|
}
|
|
728
|
-
return { text, items, method: "form", pdfClass: "E" };
|
|
791
|
+
return { kind: "extracted", text, items, method: "form", pdfClass: "E" };
|
|
729
792
|
}
|
|
730
793
|
function shapeTables(layer) {
|
|
731
794
|
const pages = layer.pages.map((page) => {
|
|
@@ -748,7 +811,7 @@ function shapeTables(layer) {
|
|
|
748
811
|
}
|
|
749
812
|
}
|
|
750
813
|
}
|
|
751
|
-
return { text, items, method: "table", pdfClass: "D" };
|
|
814
|
+
return { kind: "extracted", text, items, method: "table", pdfClass: "D" };
|
|
752
815
|
}
|
|
753
816
|
var pdfExtractor = {
|
|
754
817
|
// Every non-declined PDF extraction carries positioned runs — native text
|
|
@@ -759,25 +822,29 @@ var pdfExtractor = {
|
|
|
759
822
|
if (hit) return hit;
|
|
760
823
|
const outcome = await extractPdf(content);
|
|
761
824
|
if (cache) {
|
|
762
|
-
|
|
763
|
-
|
|
825
|
+
try {
|
|
826
|
+
if (outcome.kind === "declined") await cache.store.write(cache.key, outcome);
|
|
827
|
+
else if (outcome.items) await cache.store.write(cache.key, { ...outcome, items: outcome.items });
|
|
828
|
+
} catch {
|
|
829
|
+
}
|
|
764
830
|
}
|
|
765
831
|
return outcome;
|
|
766
832
|
}
|
|
767
833
|
};
|
|
768
834
|
async function extractPdf(content) {
|
|
769
|
-
if (!withinByteBudget(content.length)) return { declined: "too-large" };
|
|
835
|
+
if (!withinByteBudget(content.length)) return { kind: "declined", declined: "too-large" };
|
|
770
836
|
let layer;
|
|
771
837
|
try {
|
|
772
838
|
layer = await extractPdfTextLayer(content);
|
|
773
839
|
} catch (error) {
|
|
774
|
-
return { declined: classifyPdfError(error) };
|
|
840
|
+
return { kind: "declined", declined: classifyPdfError(error) };
|
|
775
841
|
}
|
|
776
842
|
if (!layer) {
|
|
777
843
|
const ocr2 = await ocrPages(content);
|
|
778
|
-
if (!ocr2.text) return { declined: "no-text-layer" };
|
|
844
|
+
if (!ocr2.text) return { kind: "declined", declined: "no-text-layer" };
|
|
779
845
|
const confidence2 = summarize(ocr2.confidences);
|
|
780
846
|
return {
|
|
847
|
+
kind: "extracted",
|
|
781
848
|
text: ocr2.text,
|
|
782
849
|
items: ocr2.items,
|
|
783
850
|
method: "ocr",
|
|
@@ -785,7 +852,7 @@ async function extractPdf(content) {
|
|
|
785
852
|
...confidence2 ? { ocrConfidence: confidence2 } : {}
|
|
786
853
|
};
|
|
787
854
|
}
|
|
788
|
-
const shaped = layer.fields.length > 0 ? foldFormFields(layer) : shapeTables(layer) ?? { text: layer.text, items: layer.items, method: "pdf-text-layer", pdfClass: "A" };
|
|
855
|
+
const shaped = layer.fields.length > 0 ? foldFormFields(layer) : shapeTables(layer) ?? { kind: "extracted", text: layer.text, items: layer.items, method: "pdf-text-layer", pdfClass: "A" };
|
|
789
856
|
const unreadPages = layer.pages.filter((page) => !page.hasTextLayer).map((page) => page.pageNumber);
|
|
790
857
|
if (unreadPages.length === 0) return shaped;
|
|
791
858
|
const recovered = await ocrPages(content, unreadPages);
|
|
@@ -818,7 +885,7 @@ async function extractPdf(content) {
|
|
|
818
885
|
var passthroughExtractor = {
|
|
819
886
|
yieldsGeometry: false,
|
|
820
887
|
async extract(content, mediaType) {
|
|
821
|
-
return { text: decodeRepresentation(content, mediaType), method: "text-passthrough" };
|
|
888
|
+
return { kind: "extracted", text: decodeRepresentation(content, mediaType), method: "text-passthrough" };
|
|
822
889
|
}
|
|
823
890
|
};
|
|
824
891
|
var EXTRACTORS = {
|
|
@@ -896,9 +963,9 @@ function createAnchoredTextStore(dir, logger) {
|
|
|
896
963
|
...hit ? "declined" in hit ? { declined: hit.declined } : { lines: hit.lines.length } : {}
|
|
897
964
|
});
|
|
898
965
|
if (!hit) return null;
|
|
899
|
-
if ("declined" in hit) return { declined: hit.declined };
|
|
966
|
+
if ("declined" in hit) return { kind: "declined", declined: hit.declined };
|
|
900
967
|
const { v: _v, stamp: _stamp, lines, text, ...provenance } = hit;
|
|
901
|
-
return { text, items: decodeLines(lines), ...provenance };
|
|
968
|
+
return { kind: "extracted", text, items: decodeLines(lines), ...provenance };
|
|
902
969
|
},
|
|
903
970
|
async write(key, outcome) {
|
|
904
971
|
const target = fileFor(key);
|
|
@@ -906,8 +973,8 @@ function createAnchoredTextStore(dir, logger) {
|
|
|
906
973
|
logger?.debug("Anchored-text cache: refusing invalid key", { key });
|
|
907
974
|
return;
|
|
908
975
|
}
|
|
909
|
-
const entry = "declined"
|
|
910
|
-
const { text, items, ...provenance } = outcome;
|
|
976
|
+
const entry = outcome.kind === "declined" ? { v: 2, stamp: STAMP, declined: outcome.declined } : (() => {
|
|
977
|
+
const { kind: _kind, text, items, ...provenance } = outcome;
|
|
911
978
|
return { v: 2, stamp: STAMP, text, lines: encodeLines(items), ...provenance };
|
|
912
979
|
})();
|
|
913
980
|
const temp = `${target}.${process.pid}.tmp`;
|
|
@@ -915,9 +982,10 @@ function createAnchoredTextStore(dir, logger) {
|
|
|
915
982
|
await fs2.promises.mkdir(path3.dirname(target), { recursive: true });
|
|
916
983
|
await fs2.promises.writeFile(temp, JSON.stringify(entry), "utf8");
|
|
917
984
|
await fs2.promises.rename(temp, target);
|
|
918
|
-
} catch {
|
|
985
|
+
} catch (error) {
|
|
919
986
|
await fs2.promises.rm(temp, { force: true }).catch(() => {
|
|
920
987
|
});
|
|
988
|
+
throw error;
|
|
921
989
|
}
|
|
922
990
|
},
|
|
923
991
|
async list() {
|
|
@@ -987,39 +1055,40 @@ function createAnchoredTextStore(dir, logger) {
|
|
|
987
1055
|
};
|
|
988
1056
|
}
|
|
989
1057
|
|
|
990
|
-
// src/
|
|
991
|
-
|
|
1058
|
+
// src/representation-reads.ts
|
|
1059
|
+
import { archivistEndpoint } from "@semiont/core/node";
|
|
1060
|
+
var RepresentationMissing = class extends Error {
|
|
1061
|
+
constructor(resourceId, reason) {
|
|
1062
|
+
super(
|
|
1063
|
+
reason === "resource" ? `Resource not found: ${resourceId}` : `Resource representation not found: no storageUri for ${resourceId}`
|
|
1064
|
+
);
|
|
1065
|
+
this.resourceId = resourceId;
|
|
1066
|
+
this.reason = reason;
|
|
1067
|
+
this.name = "RepresentationMissing";
|
|
1068
|
+
}
|
|
1069
|
+
resourceId;
|
|
1070
|
+
reason;
|
|
1071
|
+
};
|
|
1072
|
+
function archivistContentReads(config) {
|
|
1073
|
+
const { base, headers } = archivistEndpoint(config);
|
|
992
1074
|
return {
|
|
993
|
-
async
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
}
|
|
1003
|
-
},
|
|
1004
|
-
async write(key, outcome) {
|
|
1005
|
-
try {
|
|
1006
|
-
await content.putAnchoredText(key, outcome);
|
|
1007
|
-
} catch (error) {
|
|
1008
|
-
logger?.debug("Anchored-text cache: transport write failed \u2014 entry not stored", {
|
|
1009
|
-
key,
|
|
1010
|
-
reason: error instanceof Error ? error.message : String(error)
|
|
1011
|
-
});
|
|
1075
|
+
getBinary: async (resourceId) => {
|
|
1076
|
+
const url = `${base}/resources/${encodeURIComponent(String(resourceId))}/content`;
|
|
1077
|
+
const res = await fetch(url, { headers });
|
|
1078
|
+
if (res.status === 404) {
|
|
1079
|
+
const { reason } = await res.json().catch(() => ({}));
|
|
1080
|
+
throw new RepresentationMissing(
|
|
1081
|
+
String(resourceId),
|
|
1082
|
+
reason === "representation" ? "representation" : "resource"
|
|
1083
|
+
);
|
|
1012
1084
|
}
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
try {
|
|
1016
|
-
return await content.listAnchoredTextKeys();
|
|
1017
|
-
} catch (error) {
|
|
1018
|
-
logger?.debug("Anchored-text cache: transport list failed \u2014 treating as empty", {
|
|
1019
|
-
reason: error instanceof Error ? error.message : String(error)
|
|
1020
|
-
});
|
|
1021
|
-
return [];
|
|
1085
|
+
if (!res.ok) {
|
|
1086
|
+
throw new Error(`Archivist content read failed for ${String(resourceId)}: ${res.status} ${res.statusText}`);
|
|
1022
1087
|
}
|
|
1088
|
+
return {
|
|
1089
|
+
data: await res.arrayBuffer(),
|
|
1090
|
+
contentType: res.headers.get("content-type") || "application/octet-stream"
|
|
1091
|
+
};
|
|
1023
1092
|
}
|
|
1024
1093
|
};
|
|
1025
1094
|
}
|
|
@@ -1027,11 +1096,11 @@ export {
|
|
|
1027
1096
|
ChecksumMismatchError,
|
|
1028
1097
|
EXTRACTORS,
|
|
1029
1098
|
MAX_PDF_BYTES,
|
|
1099
|
+
RepresentationMissing,
|
|
1030
1100
|
WorkingTreeStore,
|
|
1031
|
-
|
|
1101
|
+
archivistContentReads,
|
|
1032
1102
|
calculateChecksum,
|
|
1033
1103
|
createAnchoredTextStore,
|
|
1034
|
-
deriveStorageUri,
|
|
1035
1104
|
extractPdfTextLayer,
|
|
1036
1105
|
verifyChecksum,
|
|
1037
1106
|
withinByteBudget
|