@jinn-network/core 0.1.2-canary.bebeb033 → 0.1.2-canary.c8da24a8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/corpus-read/acquire.d.ts +6 -0
- package/dist/corpus-read/acquire.js +69 -97
- package/dist/corpus-read/artifact-retrieval.d.ts +181 -0
- package/dist/corpus-read/artifact-retrieval.js +268 -0
- package/dist/corpus-read/cache.d.ts +7 -8
- package/dist/corpus-read/cache.js +1 -17
- package/dist/corpus-read/env.d.ts +9 -0
- package/dist/corpus-read/env.js +15 -0
- package/dist/corpus-read/fetch-artifact.js +1 -14
- package/dist/corpus-read/index.d.ts +1 -1
- package/dist/corpus-read/index.js +1 -1
- package/dist/corpus-read/ipfs.d.ts +39 -1
- package/dist/corpus-read/ipfs.js +155 -15
- package/package.json +2 -2
|
@@ -3,6 +3,12 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Always hash-verifies before persisting to cache.
|
|
5
5
|
*
|
|
6
|
+
* The two *network* legs — the donated IPFS mirror and the operator origin —
|
|
7
|
+
* are not implemented here: they are `fetchVerifiedArtifact`, the keyless
|
|
8
|
+
* retrieval primitive (#4179). What remains in this file is the store-aware
|
|
9
|
+
* composition around it: the cache read, the self-store mirror, the route
|
|
10
|
+
* resolver, and the cache writes whose ordering the daemon owns.
|
|
11
|
+
*
|
|
6
12
|
* Spec §2.3 step 4-6.
|
|
7
13
|
*/
|
|
8
14
|
import { type AcquireResult } from './fetch-artifact.js';
|
|
@@ -3,72 +3,68 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Always hash-verifies before persisting to cache.
|
|
5
5
|
*
|
|
6
|
+
* The two *network* legs — the donated IPFS mirror and the operator origin —
|
|
7
|
+
* are not implemented here: they are `fetchVerifiedArtifact`, the keyless
|
|
8
|
+
* retrieval primitive (#4179). What remains in this file is the store-aware
|
|
9
|
+
* composition around it: the cache read, the self-store mirror, the route
|
|
10
|
+
* resolver, and the cache writes whose ordering the daemon owns.
|
|
11
|
+
*
|
|
6
12
|
* Spec §2.3 step 4-6.
|
|
7
13
|
*/
|
|
8
|
-
import { createHash } from 'node:crypto';
|
|
9
14
|
import { fetchArtifactContent } from './fetch-artifact.js';
|
|
15
|
+
import { fetchVerifiedArtifact, verifyArtifactDigest } from './artifact-retrieval.js';
|
|
10
16
|
import { AcquireError, HashMismatchError } from './types.js';
|
|
11
|
-
import { classifyIpfsFetchFailure, fetchFromIpfs as defaultFetchFromIpfs } from './ipfs.js';
|
|
12
|
-
const DONATION_ARTIFACT_ENCODING = 'jinn.artifact.donation.v1';
|
|
13
|
-
function sha256Hex(buf) {
|
|
14
|
-
return createHash('sha256').update(buf).digest('hex');
|
|
15
|
-
}
|
|
16
17
|
/**
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
18
|
+
* Adapt a legacy `AcquireFn` to the primitive's clean seam. The legacy shape —
|
|
19
|
+
* a third `privateKey` parameter and a `Buffer | null` return — exists for
|
|
20
|
+
* older test fakes and stays confined to this file.
|
|
20
21
|
*/
|
|
21
|
-
function
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
|
|
31
|
-
throw new Error('donation artifact payload is not an object');
|
|
32
|
-
}
|
|
33
|
-
const record = raw;
|
|
34
|
-
if (record['schemaVersion'] !== DONATION_ARTIFACT_ENCODING || record['encoding'] !== DONATION_ARTIFACT_ENCODING) {
|
|
35
|
-
throw new Error('donation artifact payload has unexpected encoding');
|
|
36
|
-
}
|
|
37
|
-
if (record['sha256'] !== expectedSha256) {
|
|
38
|
-
throw new Error('donation artifact sha256 does not match requested artifact');
|
|
39
|
-
}
|
|
40
|
-
if (typeof record['data'] !== 'string') {
|
|
41
|
-
throw new Error('donation artifact payload is missing base64 data');
|
|
42
|
-
}
|
|
43
|
-
return Buffer.from(record['data'], 'base64');
|
|
22
|
+
function toAcquireResultFn(acquireFn, privateKey) {
|
|
23
|
+
return async (endpoint, sha256) => {
|
|
24
|
+
const raw = await acquireFn(endpoint, sha256, privateKey);
|
|
25
|
+
if (raw === null)
|
|
26
|
+
return { ok: false, reason: 'not_found', message: 'origin returned null (404 / not found)' };
|
|
27
|
+
if (Buffer.isBuffer(raw))
|
|
28
|
+
return { ok: true, content: raw };
|
|
29
|
+
return raw;
|
|
30
|
+
};
|
|
44
31
|
}
|
|
45
32
|
export async function acquireArtifactContent(args) {
|
|
46
|
-
const { sha256, artifactType, access, store, selfSafeAddress, privateKey, routeResolver, envelopeCid, sources = [], ipfsGatewayUrl, ownerSafe, acquireFn = (endpoint, sha256) => fetchArtifactContent(endpoint, sha256), fetchFromIpfs
|
|
33
|
+
const { sha256, artifactType, access, store, selfSafeAddress, privateKey, routeResolver, envelopeCid, sources = [], ipfsGatewayUrl, ownerSafe, acquireFn = (endpoint, sha256) => fetchArtifactContent(endpoint, sha256), fetchFromIpfs, } = args;
|
|
47
34
|
const now = () => new Date().toISOString();
|
|
48
|
-
// 1. Cache hit
|
|
35
|
+
// 1. Cache hit. network_artifacts is a cache, not authoritative local state:
|
|
36
|
+
// a row that does not hash to its own key is a MISS, and every leg below
|
|
37
|
+
// re-saves through INSERT OR REPLACE, so falling through repairs it. (The
|
|
38
|
+
// self-store path at step 3 throws instead, and rightly — there is no
|
|
39
|
+
// upstream to re-fetch a corrupt served_artifacts row from.)
|
|
49
40
|
const cached = store.getNetworkArtifact(sha256);
|
|
50
41
|
if (cached) {
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
42
|
+
// Verified before the usage bump: a row we are about to discard must not
|
|
43
|
+
// have its last_used_at refreshed.
|
|
44
|
+
const verifiedCache = verifyArtifactDigest(sha256, cached.content);
|
|
45
|
+
if (verifiedCache.ok) {
|
|
46
|
+
store.touchNetworkArtifactUsage(sha256, now());
|
|
47
|
+
return {
|
|
48
|
+
sha256,
|
|
49
|
+
bytes: cached.content,
|
|
50
|
+
artifactType: cached.artifactType,
|
|
51
|
+
source: 'cache',
|
|
52
|
+
paidAmountUsdc: '0',
|
|
53
|
+
fetchedAt: cached.fetchedAt,
|
|
54
|
+
sourceOperator: cached.sourceOperator ?? undefined,
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
console.warn(`[corpus-read] cached artifact ${sha256} hashed to ${verifiedCache.actualSha256}; `
|
|
58
|
+
+ 'treating the row as a miss and re-fetching');
|
|
61
59
|
}
|
|
62
|
-
// 2. Public IPFS donation source.
|
|
60
|
+
// 2. Public IPFS donation source. The primitive owns the fetch, the decode,
|
|
61
|
+
// the verification, and the #3441 fall-through warning; the cache write and
|
|
62
|
+
// the mismatch-is-fatal decision stay here, where the store lives.
|
|
63
63
|
const ipfsSource = sources.find((source) => source.kind === 'ipfs');
|
|
64
64
|
if (ipfsSource && ipfsGatewayUrl) {
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
const bytes =
|
|
68
|
-
const actualSha = sha256Hex(bytes);
|
|
69
|
-
if (actualSha !== sha256) {
|
|
70
|
-
throw new HashMismatchError(sha256, actualSha, 'ipfs', ownerSafe);
|
|
71
|
-
}
|
|
65
|
+
const retrieved = await fetchVerifiedArtifact({ sha256, artifactType }, { sources, ipfsGatewayUrl, ownerSafe, envelopeCid }, { deps: { fetchFromIpfs } });
|
|
66
|
+
if (retrieved.ok) {
|
|
67
|
+
const bytes = retrieved.artifact.bytes;
|
|
72
68
|
const ts = now();
|
|
73
69
|
store.saveNetworkArtifact({
|
|
74
70
|
sha256,
|
|
@@ -91,18 +87,13 @@ export async function acquireArtifactContent(args) {
|
|
|
91
87
|
sourceOperator: ownerSafe,
|
|
92
88
|
};
|
|
93
89
|
}
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
throw err;
|
|
97
|
-
// Donated IPFS is an opportunistic fast path. Gateway failures and
|
|
98
|
-
// malformed donation payloads should not block the acquisition chain;
|
|
99
|
-
// only verified byte hash mismatches remain fatal.
|
|
100
|
-
//
|
|
101
|
-
// Swallowing the failure silently, though, turns a size-policy refusal or
|
|
102
|
-
// a blocked redirect into an invisible data gap (#3441). Classify before
|
|
103
|
-
// swallowing so anything that is not proven absence reaches the operator.
|
|
104
|
-
warnIpfsFallThrough(`donation artifact ${sha256}`, err);
|
|
90
|
+
if (retrieved.reason === 'digest_mismatch') {
|
|
91
|
+
throw new HashMismatchError(sha256, retrieved.mismatch.actualSha256, 'ipfs', ownerSafe);
|
|
105
92
|
}
|
|
93
|
+
// Donated IPFS is an opportunistic fast path: gateway failures and
|
|
94
|
+
// malformed donation payloads fall through to the next source. Anything
|
|
95
|
+
// that was not proven absence has already reached the operator as a
|
|
96
|
+
// warning from inside the primitive (#3441).
|
|
106
97
|
}
|
|
107
98
|
// 3. Self-store fast path
|
|
108
99
|
if (ownerSafe && ownerSafe.toLowerCase() === selfSafeAddress.toLowerCase()) {
|
|
@@ -113,9 +104,9 @@ export async function acquireArtifactContent(args) {
|
|
|
113
104
|
// propagate the bad bytes into network_artifacts where peers can
|
|
114
105
|
// fetch them. Throwing closes the cache-poisoning gap; the self-store
|
|
115
106
|
// path is now hash-equivalent to origin / route-resolver.
|
|
116
|
-
const
|
|
117
|
-
if (
|
|
118
|
-
throw new HashMismatchError(sha256,
|
|
107
|
+
const verified = verifyArtifactDigest(sha256, own.content);
|
|
108
|
+
if (!verified.ok) {
|
|
109
|
+
throw new HashMismatchError(sha256, verified.actualSha256, 'self-store', selfSafeAddress);
|
|
119
110
|
}
|
|
120
111
|
// Mirror into cache so peer asks for the same content can hit cache (provenance: self-store-mirror).
|
|
121
112
|
const ts = now();
|
|
@@ -143,9 +134,9 @@ export async function acquireArtifactContent(args) {
|
|
|
143
134
|
try {
|
|
144
135
|
const out = await routeResolver.resolve({ sha256, access, requesterSafe: selfSafeAddress });
|
|
145
136
|
if (out) {
|
|
146
|
-
const
|
|
147
|
-
if (
|
|
148
|
-
throw new HashMismatchError(sha256,
|
|
137
|
+
const verified = verifyArtifactDigest(sha256, out.bytes);
|
|
138
|
+
if (!verified.ok) {
|
|
139
|
+
throw new HashMismatchError(sha256, verified.actualSha256, 'route-resolver', out.sourceOperator);
|
|
149
140
|
}
|
|
150
141
|
const ts = now();
|
|
151
142
|
store.saveNetworkArtifact({
|
|
@@ -176,36 +167,17 @@ export async function acquireArtifactContent(args) {
|
|
|
176
167
|
}
|
|
177
168
|
}
|
|
178
169
|
// 5. Origin fetch
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
// union from fetchArtifactContent. Buffer.isBuffer guards before we
|
|
184
|
-
// duck-type into the union.
|
|
185
|
-
if (raw === null || Buffer.isBuffer(raw)) {
|
|
186
|
-
bytes = raw;
|
|
170
|
+
const retrieved = await fetchVerifiedArtifact({ sha256, artifactType }, { endpoint: access.endpoint, ownerSafe, envelopeCid }, { deps: { fetchArtifact: toAcquireResultFn(acquireFn, privateKey) } });
|
|
171
|
+
if (!retrieved.ok) {
|
|
172
|
+
if (retrieved.reason === 'digest_mismatch') {
|
|
173
|
+
throw new HashMismatchError(sha256, retrieved.mismatch.actualSha256, 'origin', ownerSafe);
|
|
187
174
|
}
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
// ok: false — preserve the reason so the caller / daemon route can
|
|
193
|
-
// surface it via AcquireError.cause.
|
|
194
|
-
throw new AcquireError(sha256, `origin fetch failed: ${raw.reason}${raw.message ? ` (${raw.message})` : ''}`, raw);
|
|
195
|
-
}
|
|
196
|
-
}
|
|
197
|
-
catch (err) {
|
|
198
|
-
if (err instanceof AcquireError)
|
|
199
|
-
throw err;
|
|
200
|
-
throw new AcquireError(sha256, 'origin fetch failed', err);
|
|
201
|
-
}
|
|
202
|
-
if (!bytes) {
|
|
203
|
-
throw new AcquireError(sha256, 'origin returned null (404 / not found)');
|
|
204
|
-
}
|
|
205
|
-
const actualSha = sha256Hex(bytes);
|
|
206
|
-
if (actualSha !== sha256) {
|
|
207
|
-
throw new HashMismatchError(sha256, actualSha, 'origin', ownerSafe);
|
|
175
|
+
// Preserve the reason token in the message so callers and the daemon route
|
|
176
|
+
// can surface which failure this was.
|
|
177
|
+
const attempt = retrieved.attempts.find((entry) => entry.leg === 'origin');
|
|
178
|
+
throw new AcquireError(sha256, `origin fetch failed: ${retrieved.reason}${attempt?.message ? ` (${attempt.message})` : ''}`, retrieved);
|
|
208
179
|
}
|
|
180
|
+
const bytes = retrieved.artifact.bytes;
|
|
209
181
|
const ts = now();
|
|
210
182
|
store.saveNetworkArtifact({
|
|
211
183
|
sha256,
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Artifact retrieval primitive — fetch by content address, verify, return bytes.
|
|
3
|
+
*
|
|
4
|
+
* One function fetches an artifact from the network legs an envelope names and
|
|
5
|
+
* hands back its bytes only after they hash to the recorded sha256, together
|
|
6
|
+
* with the provenance of the leg that produced them. A digest mismatch fails
|
|
7
|
+
* closed: the failure arm of the result union has no bytes on it at all.
|
|
8
|
+
*
|
|
9
|
+
* Keyless and filesystem-neutral by construction. The primitive itself neither
|
|
10
|
+
* accepts nor touches key material or the filesystem: it takes no store, no
|
|
11
|
+
* signer, no private key, and no path, and it performs no write of any kind.
|
|
12
|
+
* The one channel through which such material can reach a call is the caller's
|
|
13
|
+
* own `deps` seam closure — `acquire.ts` legitimately supplies one that holds a
|
|
14
|
+
* private key — and that closure is opaque here: it is invoked, never read,
|
|
15
|
+
* never logged, and never surfaced in provenance.
|
|
16
|
+
* `test/architecture/keyless-primitive.test.ts` holds the module graph to the
|
|
17
|
+
* same promise.
|
|
18
|
+
*
|
|
19
|
+
* Where the returned bytes land is the caller's decision. The daemon chain
|
|
20
|
+
* (`acquire.ts`) composes this with its byte cache; the CLI hands them to the
|
|
21
|
+
* operator; the MCP handler proxies to a daemon that runs it. None of that is
|
|
22
|
+
* this function's business.
|
|
23
|
+
*
|
|
24
|
+
* Issue #4179 (umbrella #2448 / journey spec B0c).
|
|
25
|
+
*/
|
|
26
|
+
import { type AcquireResult } from './fetch-artifact.js';
|
|
27
|
+
import type { ArtifactSource } from './types.js';
|
|
28
|
+
/**
|
|
29
|
+
* Content address of a corpus artifact: the sha256 of the bytes themselves, as
|
|
30
|
+
* recorded in the envelope's `artifact.sha256` — never an IPFS CID (artifacts
|
|
31
|
+
* stopped having CIDs post jinn-mono-vy37.1.2).
|
|
32
|
+
*/
|
|
33
|
+
export interface ArtifactAddress {
|
|
34
|
+
/** 64-char lowercase hex. The one thing that is trusted. */
|
|
35
|
+
sha256: string;
|
|
36
|
+
/**
|
|
37
|
+
* Declared type from the envelope. Carried into the result for the caller's
|
|
38
|
+
* convenience and never consulted during verification — an attacker-supplied
|
|
39
|
+
* label cannot influence whether bytes are admitted.
|
|
40
|
+
*/
|
|
41
|
+
artifactType?: string;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Where the bytes might be. Every field is manifest-supplied and therefore
|
|
45
|
+
* attacker-controlled; the destination policy and the byte/redirect/deadline
|
|
46
|
+
* caps in `fetch-artifact.ts` and `ipfs.ts` are what make that safe.
|
|
47
|
+
*/
|
|
48
|
+
export interface ArtifactLocators {
|
|
49
|
+
/** Donated IPFS sources — the envelope's `artifact.sources` entries. */
|
|
50
|
+
sources?: readonly ArtifactSource[];
|
|
51
|
+
/** Gateway base for the IPFS leg. The leg is skipped when this is absent. */
|
|
52
|
+
ipfsGatewayUrl?: string;
|
|
53
|
+
/** HTTP origin — the envelope's `artifact.access.endpoint`. */
|
|
54
|
+
endpoint?: string;
|
|
55
|
+
/** Envelope CID this artifact was named by. Provenance only. */
|
|
56
|
+
envelopeCid?: string;
|
|
57
|
+
/** Publishing operator's Safe, when known. Provenance and attribution only. */
|
|
58
|
+
ownerSafe?: string;
|
|
59
|
+
}
|
|
60
|
+
export interface FetchVerifiedArtifactOptions {
|
|
61
|
+
/**
|
|
62
|
+
* Injection seams, mirroring the `acquire.ts` precedent so tests need no
|
|
63
|
+
* module mock. A caller that needs to bound or redirect a leg replaces the
|
|
64
|
+
* leg — there is no separate options passthrough, because no call site wants
|
|
65
|
+
* the default transport with different bounds.
|
|
66
|
+
*/
|
|
67
|
+
deps?: {
|
|
68
|
+
fetchArtifact?: (endpoint: string, sha256: string) => Promise<AcquireResult>;
|
|
69
|
+
fetchFromIpfs?: (gatewayUrl: string, cid: string) => Promise<unknown>;
|
|
70
|
+
/** Clock seam for `fetchedAt`. */
|
|
71
|
+
now?: () => string;
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
/** One leg's outcome, recorded whether it succeeded, was skipped, or failed. */
|
|
75
|
+
export interface ArtifactFetchAttempt {
|
|
76
|
+
readonly leg: 'ipfs' | 'origin';
|
|
77
|
+
/** `ipfs://<cid>` or the exact URL `buildArtifactUrl` produced; empty when skipped. */
|
|
78
|
+
readonly sourceUri: string;
|
|
79
|
+
readonly outcome: 'verified' | 'skipped' | 'failed' | 'digest_mismatch';
|
|
80
|
+
/**
|
|
81
|
+
* Present on `failed`; the classified reason, never a raw string match.
|
|
82
|
+
* `digest_mismatch` is excluded because `refuse()` records that outcome with
|
|
83
|
+
* no `reason` field — no attempt ever carries the token, and saying so here
|
|
84
|
+
* is what lets the failure union below narrow without a cast.
|
|
85
|
+
*/
|
|
86
|
+
readonly reason?: Exclude<ArtifactFetchFailureReason, 'digest_mismatch'>;
|
|
87
|
+
readonly message?: string;
|
|
88
|
+
}
|
|
89
|
+
export interface ArtifactProvenance {
|
|
90
|
+
/** Which leg produced the admitted bytes. */
|
|
91
|
+
readonly source: 'ipfs' | 'origin';
|
|
92
|
+
readonly sourceUri: string;
|
|
93
|
+
/** Always 'sha256'. Explicit so the record does not silently mean "some digest". */
|
|
94
|
+
readonly digestAlgorithm: 'sha256';
|
|
95
|
+
/** ISO-8601 instant the bytes were verified, not when they were requested. */
|
|
96
|
+
readonly fetchedAt: string;
|
|
97
|
+
readonly sourceOperator?: string;
|
|
98
|
+
readonly envelopeCid?: string;
|
|
99
|
+
/** Every leg tried, in order. Legible even on the happy path. */
|
|
100
|
+
readonly attempts: readonly ArtifactFetchAttempt[];
|
|
101
|
+
}
|
|
102
|
+
export interface VerifiedArtifact {
|
|
103
|
+
readonly sha256: string;
|
|
104
|
+
/** Verified bytes, in memory. Never a path, never a handle. */
|
|
105
|
+
readonly bytes: Buffer;
|
|
106
|
+
readonly artifactType: string;
|
|
107
|
+
readonly provenance: ArtifactProvenance;
|
|
108
|
+
}
|
|
109
|
+
export type ArtifactFetchFailureReason =
|
|
110
|
+
/** Bytes arrived and did not hash to the address. */
|
|
111
|
+
'digest_mismatch'
|
|
112
|
+
/** Every attempted leg answered, and every answer was absence. */
|
|
113
|
+
| 'not_found'
|
|
114
|
+
/** Nothing was supplied to try. */
|
|
115
|
+
| 'no_locator'
|
|
116
|
+
/** Destination policy refused an origin or a redirect hop. */
|
|
117
|
+
| 'blocked' | 'too_large' | 'timeout'
|
|
118
|
+
/** Transport failure — nothing was learned about whether the content exists. */
|
|
119
|
+
| 'unavailable'
|
|
120
|
+
/**
|
|
121
|
+
* A locator resolved and what it served is permanently not this artifact —
|
|
122
|
+
* a broken donation payload. Conclusive absence AT THAT LOCATOR, so it is
|
|
123
|
+
* not retryable; never conclusive globally, so it never outranks a leg that
|
|
124
|
+
* learned nothing (#3441).
|
|
125
|
+
*/
|
|
126
|
+
| 'malformed_payload';
|
|
127
|
+
/** Evidence, not content. Present on `digest_mismatch` and only there. */
|
|
128
|
+
export interface ArtifactDigestMismatch {
|
|
129
|
+
readonly expectedSha256: string;
|
|
130
|
+
readonly actualSha256: string;
|
|
131
|
+
readonly sourceUri: string;
|
|
132
|
+
readonly sourceOperator?: string;
|
|
133
|
+
}
|
|
134
|
+
interface FetchVerifiedArtifactFailureBase {
|
|
135
|
+
readonly ok: false;
|
|
136
|
+
readonly sha256: string;
|
|
137
|
+
/** Derived from `reason`, never caller-supplied. */
|
|
138
|
+
readonly retryable: boolean;
|
|
139
|
+
readonly message: string;
|
|
140
|
+
readonly attempts: readonly ArtifactFetchAttempt[];
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Discriminated on `reason` so the compiler, not a constructor comment, carries
|
|
144
|
+
* the invariant that `digest_mismatch` always ships its evidence. `refuse()` is
|
|
145
|
+
* the sole producer of that reason and has always set `mismatch`; before this
|
|
146
|
+
* split, callers reading the evidence had to assert it with `!`.
|
|
147
|
+
*/
|
|
148
|
+
export type FetchVerifiedArtifactFailure = (FetchVerifiedArtifactFailureBase & {
|
|
149
|
+
readonly reason: 'digest_mismatch';
|
|
150
|
+
readonly mismatch: ArtifactDigestMismatch;
|
|
151
|
+
}) | (FetchVerifiedArtifactFailureBase & {
|
|
152
|
+
readonly reason: Exclude<ArtifactFetchFailureReason, 'digest_mismatch'>;
|
|
153
|
+
readonly mismatch?: undefined;
|
|
154
|
+
});
|
|
155
|
+
export type FetchVerifiedArtifactResult = {
|
|
156
|
+
readonly ok: true;
|
|
157
|
+
readonly artifact: VerifiedArtifact;
|
|
158
|
+
} | FetchVerifiedArtifactFailure;
|
|
159
|
+
/**
|
|
160
|
+
* The one digest check in the system. Legs that already hold bytes — the byte
|
|
161
|
+
* cache, the self-store, a route resolver, the MCP daemon proxy — call this
|
|
162
|
+
* rather than writing `createHash('sha256')` out again.
|
|
163
|
+
*/
|
|
164
|
+
export declare function verifyArtifactDigest(expectedSha256: string, bytes: Buffer): {
|
|
165
|
+
ok: true;
|
|
166
|
+
} | {
|
|
167
|
+
ok: false;
|
|
168
|
+
actualSha256: string;
|
|
169
|
+
};
|
|
170
|
+
/**
|
|
171
|
+
* Fetch an artifact by content address and return its bytes only after they
|
|
172
|
+
* hash to that address.
|
|
173
|
+
*
|
|
174
|
+
* Legs run in the order the daemon chain has always used: the free donated IPFS
|
|
175
|
+
* mirror first, the operator's HTTP origin second. A leg with no locator is
|
|
176
|
+
* *skipped*, not failed. A digest mismatch on any leg fails the whole call and
|
|
177
|
+
* does not fall through — the bytes are wrong or hostile, and the next locator
|
|
178
|
+
* cannot make them right.
|
|
179
|
+
*/
|
|
180
|
+
export declare function fetchVerifiedArtifact(address: ArtifactAddress, locators: ArtifactLocators, options?: FetchVerifiedArtifactOptions): Promise<FetchVerifiedArtifactResult>;
|
|
181
|
+
export {};
|
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Artifact retrieval primitive — fetch by content address, verify, return bytes.
|
|
3
|
+
*
|
|
4
|
+
* One function fetches an artifact from the network legs an envelope names and
|
|
5
|
+
* hands back its bytes only after they hash to the recorded sha256, together
|
|
6
|
+
* with the provenance of the leg that produced them. A digest mismatch fails
|
|
7
|
+
* closed: the failure arm of the result union has no bytes on it at all.
|
|
8
|
+
*
|
|
9
|
+
* Keyless and filesystem-neutral by construction. The primitive itself neither
|
|
10
|
+
* accepts nor touches key material or the filesystem: it takes no store, no
|
|
11
|
+
* signer, no private key, and no path, and it performs no write of any kind.
|
|
12
|
+
* The one channel through which such material can reach a call is the caller's
|
|
13
|
+
* own `deps` seam closure — `acquire.ts` legitimately supplies one that holds a
|
|
14
|
+
* private key — and that closure is opaque here: it is invoked, never read,
|
|
15
|
+
* never logged, and never surfaced in provenance.
|
|
16
|
+
* `test/architecture/keyless-primitive.test.ts` holds the module graph to the
|
|
17
|
+
* same promise.
|
|
18
|
+
*
|
|
19
|
+
* Where the returned bytes land is the caller's decision. The daemon chain
|
|
20
|
+
* (`acquire.ts`) composes this with its byte cache; the CLI hands them to the
|
|
21
|
+
* operator; the MCP handler proxies to a daemon that runs it. None of that is
|
|
22
|
+
* this function's business.
|
|
23
|
+
*
|
|
24
|
+
* Issue #4179 (umbrella #2448 / journey spec B0c).
|
|
25
|
+
*/
|
|
26
|
+
import { createHash } from 'node:crypto';
|
|
27
|
+
import { buildArtifactUrl, fetchArtifactContent, } from './fetch-artifact.js';
|
|
28
|
+
import { classifyIpfsFetchFailure, fetchFromIpfs as defaultFetchFromIpfs } from './ipfs.js';
|
|
29
|
+
const DONATION_ARTIFACT_ENCODING = 'jinn.artifact.donation.v1';
|
|
30
|
+
/** Nothing that was learned tells us to stop; everything else does. */
|
|
31
|
+
const RETRYABLE_REASONS = new Set([
|
|
32
|
+
'timeout',
|
|
33
|
+
'unavailable',
|
|
34
|
+
]);
|
|
35
|
+
/** Reasons that are positive proof of absence at the leg that reported them. */
|
|
36
|
+
const CONCLUSIVE_ABSENCE = new Set([
|
|
37
|
+
'not_found',
|
|
38
|
+
'malformed_payload',
|
|
39
|
+
]);
|
|
40
|
+
/**
|
|
41
|
+
* The one digest check in the system. Legs that already hold bytes — the byte
|
|
42
|
+
* cache, the self-store, a route resolver, the MCP daemon proxy — call this
|
|
43
|
+
* rather than writing `createHash('sha256')` out again.
|
|
44
|
+
*/
|
|
45
|
+
export function verifyArtifactDigest(expectedSha256, bytes) {
|
|
46
|
+
const actualSha256 = createHash('sha256').update(bytes).digest('hex');
|
|
47
|
+
return actualSha256 === expectedSha256 ? { ok: true } : { ok: false, actualSha256 };
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Log an IPFS read the retrieval is about to fall through, unless the gateway
|
|
51
|
+
* actually answered "not there" (#3441). A cap refusal is positive evidence the
|
|
52
|
+
* content exists, so it must not read like an ordinary miss.
|
|
53
|
+
*/
|
|
54
|
+
function warnIpfsFallThrough(subject, error) {
|
|
55
|
+
// A broken donation is not the ordinary absence #3441 carved out of the
|
|
56
|
+
// warning, so it still warns — but it must not read as a transport failure.
|
|
57
|
+
const classification = error instanceof DonationDecodeError
|
|
58
|
+
? 'malformed-payload'
|
|
59
|
+
: classifyIpfsFetchFailure(error);
|
|
60
|
+
if (classification === 'not-found')
|
|
61
|
+
return;
|
|
62
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
63
|
+
console.warn(`[corpus-read] IPFS source for ${subject} could not be used (${classification}), `
|
|
64
|
+
+ `falling through to the next source: ${detail}`);
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* A donation payload that will never decode into this artifact. Distinct from a
|
|
68
|
+
* gateway transport failure: the CID resolved, and what it resolved to is not
|
|
69
|
+
* the requested bytes — including a valid donation for a *different* artifact.
|
|
70
|
+
*/
|
|
71
|
+
class DonationDecodeError extends Error {
|
|
72
|
+
name = 'DonationDecodeError';
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Decode a `jinn.artifact.donation.v1` payload into its raw bytes.
|
|
76
|
+
*
|
|
77
|
+
* Moved verbatim from `acquire.ts`. The `schemaVersion` check compares against
|
|
78
|
+
* the encoding constant — the published fixtures set both fields to that same
|
|
79
|
+
* string, so the comparison is load-bearing as written and is not "corrected"
|
|
80
|
+
* here.
|
|
81
|
+
*/
|
|
82
|
+
function decodeDonationArtifact(raw, expectedSha256) {
|
|
83
|
+
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
|
|
84
|
+
throw new DonationDecodeError('donation artifact payload is not an object');
|
|
85
|
+
}
|
|
86
|
+
const record = raw;
|
|
87
|
+
if (record['schemaVersion'] !== DONATION_ARTIFACT_ENCODING || record['encoding'] !== DONATION_ARTIFACT_ENCODING) {
|
|
88
|
+
throw new DonationDecodeError('donation artifact payload has unexpected encoding');
|
|
89
|
+
}
|
|
90
|
+
if (record['sha256'] !== expectedSha256) {
|
|
91
|
+
throw new DonationDecodeError('donation artifact sha256 does not match requested artifact');
|
|
92
|
+
}
|
|
93
|
+
if (typeof record['data'] !== 'string') {
|
|
94
|
+
throw new DonationDecodeError('donation artifact payload is missing base64 data');
|
|
95
|
+
}
|
|
96
|
+
return Buffer.from(record['data'], 'base64');
|
|
97
|
+
}
|
|
98
|
+
/** `fetchArtifactContent`'s vocabulary → the primitive's. */
|
|
99
|
+
function originReason(reason) {
|
|
100
|
+
return reason === 'network_error' ? 'unavailable' : reason;
|
|
101
|
+
}
|
|
102
|
+
/** `classifyIpfsFetchFailure`'s hyphenated vocabulary → the primitive's. */
|
|
103
|
+
function ipfsReason(classification) {
|
|
104
|
+
if (classification === 'too-large')
|
|
105
|
+
return 'too_large';
|
|
106
|
+
if (classification === 'not-found')
|
|
107
|
+
return 'not_found';
|
|
108
|
+
return 'unavailable';
|
|
109
|
+
}
|
|
110
|
+
function errorMessage(error) {
|
|
111
|
+
return error instanceof Error ? error.message : String(error);
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Fetch an artifact by content address and return its bytes only after they
|
|
115
|
+
* hash to that address.
|
|
116
|
+
*
|
|
117
|
+
* Legs run in the order the daemon chain has always used: the free donated IPFS
|
|
118
|
+
* mirror first, the operator's HTTP origin second. A leg with no locator is
|
|
119
|
+
* *skipped*, not failed. A digest mismatch on any leg fails the whole call and
|
|
120
|
+
* does not fall through — the bytes are wrong or hostile, and the next locator
|
|
121
|
+
* cannot make them right.
|
|
122
|
+
*/
|
|
123
|
+
export async function fetchVerifiedArtifact(address, locators, options) {
|
|
124
|
+
const { sha256, artifactType } = address;
|
|
125
|
+
const { sources = [], ipfsGatewayUrl, endpoint, envelopeCid, ownerSafe } = locators;
|
|
126
|
+
const fetchIpfs = options?.deps?.fetchFromIpfs ?? defaultFetchFromIpfs;
|
|
127
|
+
const fetchOrigin = options?.deps?.fetchArtifact ?? fetchArtifactContent;
|
|
128
|
+
const now = options?.deps?.now ?? (() => new Date().toISOString());
|
|
129
|
+
const attempts = [];
|
|
130
|
+
const admit = (source, sourceUri, bytes) => {
|
|
131
|
+
attempts.push({ leg: source, sourceUri, outcome: 'verified' });
|
|
132
|
+
return {
|
|
133
|
+
ok: true,
|
|
134
|
+
artifact: {
|
|
135
|
+
sha256,
|
|
136
|
+
bytes,
|
|
137
|
+
artifactType: artifactType ?? 'unknown',
|
|
138
|
+
provenance: {
|
|
139
|
+
source,
|
|
140
|
+
sourceUri,
|
|
141
|
+
digestAlgorithm: 'sha256',
|
|
142
|
+
fetchedAt: now(),
|
|
143
|
+
...(ownerSafe ? { sourceOperator: ownerSafe } : {}),
|
|
144
|
+
...(envelopeCid ? { envelopeCid } : {}),
|
|
145
|
+
attempts,
|
|
146
|
+
},
|
|
147
|
+
},
|
|
148
|
+
};
|
|
149
|
+
};
|
|
150
|
+
const refuse = (leg, sourceUri, actualSha256) => {
|
|
151
|
+
attempts.push({ leg, sourceUri, outcome: 'digest_mismatch' });
|
|
152
|
+
return {
|
|
153
|
+
ok: false,
|
|
154
|
+
sha256,
|
|
155
|
+
reason: 'digest_mismatch',
|
|
156
|
+
retryable: false,
|
|
157
|
+
message: `artifact ${sha256} from ${sourceUri} hashed to ${actualSha256}; `
|
|
158
|
+
+ 'the bytes were discarded',
|
|
159
|
+
attempts,
|
|
160
|
+
mismatch: {
|
|
161
|
+
expectedSha256: sha256,
|
|
162
|
+
actualSha256,
|
|
163
|
+
sourceUri,
|
|
164
|
+
...(ownerSafe ? { sourceOperator: ownerSafe } : {}),
|
|
165
|
+
},
|
|
166
|
+
};
|
|
167
|
+
};
|
|
168
|
+
// 1. Donated IPFS mirror — free and public, so it goes first.
|
|
169
|
+
const ipfsSource = sources.find((source) => source.kind === 'ipfs');
|
|
170
|
+
if (ipfsSource && ipfsGatewayUrl) {
|
|
171
|
+
const sourceUri = `ipfs://${ipfsSource.cid}`;
|
|
172
|
+
try {
|
|
173
|
+
const raw = await fetchIpfs(ipfsGatewayUrl, ipfsSource.cid);
|
|
174
|
+
const bytes = decodeDonationArtifact(raw, sha256);
|
|
175
|
+
const verified = verifyArtifactDigest(sha256, bytes);
|
|
176
|
+
if (!verified.ok)
|
|
177
|
+
return refuse('ipfs', sourceUri, verified.actualSha256);
|
|
178
|
+
return admit('ipfs', sourceUri, bytes);
|
|
179
|
+
}
|
|
180
|
+
catch (err) {
|
|
181
|
+
// Neither a gateway failure nor a malformed donation payload is a verdict
|
|
182
|
+
// on the artifact *globally*, so both fall through and let the next
|
|
183
|
+
// locator answer. They differ in what they proved here: a transport
|
|
184
|
+
// failure learned nothing, while a payload that will never decode into
|
|
185
|
+
// this artifact is absence at this locator — hence the split reason. Both
|
|
186
|
+
// still warn; only a gateway's own "not there" is silent (#3441).
|
|
187
|
+
warnIpfsFallThrough(`donation artifact ${sha256}`, err);
|
|
188
|
+
attempts.push({
|
|
189
|
+
leg: 'ipfs',
|
|
190
|
+
sourceUri,
|
|
191
|
+
outcome: 'failed',
|
|
192
|
+
reason: err instanceof DonationDecodeError
|
|
193
|
+
? 'malformed_payload'
|
|
194
|
+
: ipfsReason(classifyIpfsFetchFailure(err)),
|
|
195
|
+
message: errorMessage(err),
|
|
196
|
+
});
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
else {
|
|
200
|
+
attempts.push({ leg: 'ipfs', sourceUri: '', outcome: 'skipped' });
|
|
201
|
+
}
|
|
202
|
+
// 2. Operator HTTP origin.
|
|
203
|
+
if (endpoint) {
|
|
204
|
+
const sourceUri = buildArtifactUrl(endpoint, sha256);
|
|
205
|
+
let outcome;
|
|
206
|
+
try {
|
|
207
|
+
outcome = await fetchOrigin(endpoint, sha256);
|
|
208
|
+
}
|
|
209
|
+
catch (err) {
|
|
210
|
+
outcome = { ok: false, reason: 'network_error', message: errorMessage(err) };
|
|
211
|
+
}
|
|
212
|
+
if (outcome.ok) {
|
|
213
|
+
const verified = verifyArtifactDigest(sha256, outcome.content);
|
|
214
|
+
if (!verified.ok)
|
|
215
|
+
return refuse('origin', sourceUri, verified.actualSha256);
|
|
216
|
+
return admit('origin', sourceUri, outcome.content);
|
|
217
|
+
}
|
|
218
|
+
attempts.push({
|
|
219
|
+
leg: 'origin',
|
|
220
|
+
sourceUri,
|
|
221
|
+
outcome: 'failed',
|
|
222
|
+
reason: originReason(outcome.reason),
|
|
223
|
+
...(outcome.message ? { message: outcome.message } : {}),
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
else {
|
|
227
|
+
attempts.push({ leg: 'origin', sourceUri: '', outcome: 'skipped' });
|
|
228
|
+
}
|
|
229
|
+
// Nothing verified. Absence is only claimed when every attempted leg said so:
|
|
230
|
+
// one inconclusive leg means we learned nothing about existence (#3441).
|
|
231
|
+
const failures = attempts.filter((attempt) => attempt.outcome === 'failed');
|
|
232
|
+
if (failures.length === 0) {
|
|
233
|
+
// Name why each leg was skipped rather than assuming both locators were
|
|
234
|
+
// absent: donated sources with no gateway to read them through is its own
|
|
235
|
+
// case, and reporting it as "no donated IPFS source" is simply wrong.
|
|
236
|
+
const missing = [
|
|
237
|
+
ipfsSource
|
|
238
|
+
? 'a donated IPFS source but no gateway URL to read it through'
|
|
239
|
+
: 'no donated IPFS source',
|
|
240
|
+
'no origin endpoint',
|
|
241
|
+
];
|
|
242
|
+
return {
|
|
243
|
+
ok: false,
|
|
244
|
+
sha256,
|
|
245
|
+
reason: 'no_locator',
|
|
246
|
+
retryable: false,
|
|
247
|
+
message: `artifact ${sha256} has no locator to try: ${missing.join(', and ')}`,
|
|
248
|
+
attempts,
|
|
249
|
+
};
|
|
250
|
+
}
|
|
251
|
+
// When more than one leg fails inconclusively, the last one wins. Legs run
|
|
252
|
+
// cheapest-first — the opportunistic donated mirror, then the operator's own
|
|
253
|
+
// origin — so the last leg to answer is the artifact's actual home, and its
|
|
254
|
+
// reason is the one the operator can act on.
|
|
255
|
+
// Absence is a set, not one literal: a leg that resolved and served bytes that
|
|
256
|
+
// are permanently not this artifact answered as conclusively as a 404 did.
|
|
257
|
+
const inconclusive = failures.filter((attempt) => !CONCLUSIVE_ABSENCE.has(attempt.reason));
|
|
258
|
+
const reason = (inconclusive.length > 0 ? inconclusive : failures).at(-1).reason;
|
|
259
|
+
return {
|
|
260
|
+
ok: false,
|
|
261
|
+
sha256,
|
|
262
|
+
reason,
|
|
263
|
+
retryable: RETRYABLE_REASONS.has(reason),
|
|
264
|
+
message: `artifact ${sha256} could not be retrieved (${reason}) from `
|
|
265
|
+
+ failures.map((attempt) => `${attempt.sourceUri}: ${attempt.reason}`).join('; '),
|
|
266
|
+
attempts,
|
|
267
|
+
};
|
|
268
|
+
}
|
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Corpus cache
|
|
2
|
+
* Corpus cache is not a public helper surface.
|
|
3
3
|
*
|
|
4
4
|
* The corpus library reuses the existing `network_artifacts` and `served_artifacts`
|
|
5
|
-
* tables on the daemon's `Store`. There is no separate corpus cache
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
5
|
+
* tables on the daemon's `Store`. There is no separate corpus cache API —
|
|
6
|
+
* `acquireArtifactContent` is the only writer/reader, and it digest-checks cached
|
|
7
|
+
* rows before serving them. Do not reintroduce `getCachedArtifact` /
|
|
8
|
+
* `hasCachedArtifact`: they returned unverified `network_artifacts` rows and
|
|
9
|
+
* would reopen the byte-admission gap #4359 closed (#4500).
|
|
9
10
|
*
|
|
10
11
|
* Spec: spec/2026-04-30-phase-a-umbrella.md §2.4 (caching semantics).
|
|
11
12
|
*/
|
|
12
|
-
|
|
13
|
-
export declare function getCachedArtifact(store: CorpusStorePort, sha256: string): NetworkArtifactRow | null;
|
|
14
|
-
export declare function hasCachedArtifact(store: CorpusStorePort, sha256: string): boolean;
|
|
13
|
+
export {};
|
|
@@ -1,17 +1 @@
|
|
|
1
|
-
|
|
2
|
-
* Corpus cache helpers.
|
|
3
|
-
*
|
|
4
|
-
* The corpus library reuses the existing `network_artifacts` and `served_artifacts`
|
|
5
|
-
* tables on the daemon's `Store`. There is no separate corpus cache layer — the
|
|
6
|
-
* acquire chain in `acquire.ts` is the only writer/reader. This module exposes
|
|
7
|
-
* thin convenience helpers for higher-level code that wants to inspect or
|
|
8
|
-
* pre-populate the cache without going through `acquireArtifactContent`.
|
|
9
|
-
*
|
|
10
|
-
* Spec: spec/2026-04-30-phase-a-umbrella.md §2.4 (caching semantics).
|
|
11
|
-
*/
|
|
12
|
-
export function getCachedArtifact(store, sha256) {
|
|
13
|
-
return store.getNetworkArtifact(sha256);
|
|
14
|
-
}
|
|
15
|
-
export function hasCachedArtifact(store, sha256) {
|
|
16
|
-
return store.getNetworkArtifact(sha256) !== null;
|
|
17
|
-
}
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/** Environment-variable parsing shared by the corpus read paths. */
|
|
2
|
+
/**
|
|
3
|
+
* `minimum` is 0 where the bound reads `0` as "disabled" (a whole-operation
|
|
4
|
+
* timeout, or a redirect cap of zero meaning "follow none"), and 1 for a byte
|
|
5
|
+
* cap, where zero would not disable anything — it would reject every response
|
|
6
|
+
* as `too_large`. A foot-gun that silently stops all acquisition is worse than
|
|
7
|
+
* ignoring the value, so an out-of-range setting falls back to the default.
|
|
8
|
+
*/
|
|
9
|
+
export declare function envInteger(name: string, fallback: number, minimum?: number): number;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/** Environment-variable parsing shared by the corpus read paths. */
|
|
2
|
+
/**
|
|
3
|
+
* `minimum` is 0 where the bound reads `0` as "disabled" (a whole-operation
|
|
4
|
+
* timeout, or a redirect cap of zero meaning "follow none"), and 1 for a byte
|
|
5
|
+
* cap, where zero would not disable anything — it would reject every response
|
|
6
|
+
* as `too_large`. A foot-gun that silently stops all acquisition is worse than
|
|
7
|
+
* ignoring the value, so an out-of-range setting falls back to the default.
|
|
8
|
+
*/
|
|
9
|
+
export function envInteger(name, fallback, minimum = 0) {
|
|
10
|
+
const raw = process.env[name];
|
|
11
|
+
if (raw === undefined || raw.trim() === '')
|
|
12
|
+
return fallback;
|
|
13
|
+
const parsed = Number(raw);
|
|
14
|
+
return Number.isInteger(parsed) && parsed >= minimum ? parsed : fallback;
|
|
15
|
+
}
|
|
@@ -23,6 +23,7 @@
|
|
|
23
23
|
* - the body is streamed through a byte counter and abandoned the moment it
|
|
24
24
|
* exceeds the cap, so an oversized response is never fully buffered.
|
|
25
25
|
*/
|
|
26
|
+
import { envInteger } from './env.js';
|
|
26
27
|
import { resolvePublicHttpDestination, ProhibitedDestinationError, } from './origin-guard.js';
|
|
27
28
|
import { pinnedFetch } from './pinned-fetch.js';
|
|
28
29
|
/** 32 MiB. Artifacts are solution/evidence payloads, not disk images. */
|
|
@@ -34,20 +35,6 @@ class TimeoutError extends Error {
|
|
|
34
35
|
}
|
|
35
36
|
class TooLargeError extends Error {
|
|
36
37
|
}
|
|
37
|
-
/**
|
|
38
|
-
* `minimum` is 0 where the bound reads `0` as "disabled" (the timeout, and a
|
|
39
|
-
* redirect cap of zero meaning "follow none"), and 1 for the byte cap, where
|
|
40
|
-
* zero would not disable anything — it would reject every artifact as
|
|
41
|
-
* `too_large`. A foot-gun that silently stops all acquisition is worse than
|
|
42
|
-
* ignoring the value, so an out-of-range setting falls back to the default.
|
|
43
|
-
*/
|
|
44
|
-
function envInteger(name, fallback, minimum = 0) {
|
|
45
|
-
const raw = process.env[name];
|
|
46
|
-
if (raw === undefined || raw.trim() === '')
|
|
47
|
-
return fallback;
|
|
48
|
-
const parsed = Number(raw);
|
|
49
|
-
return Number.isInteger(parsed) && parsed >= minimum ? parsed : fallback;
|
|
50
|
-
}
|
|
51
38
|
function envFlag(name) {
|
|
52
39
|
const raw = process.env[name];
|
|
53
40
|
if (raw === undefined)
|
|
@@ -56,6 +56,37 @@ export declare class IpfsFetchFailedError extends Error {
|
|
|
56
56
|
* default fetch also tries the `ipfs.io` fallback, which typically stalls into a 504 or an abort
|
|
57
57
|
* for an unpinned digest rather than answering 404. Such a run classifies `'unavailable'` — the
|
|
58
58
|
* safe direction, since absence is never claimed without proof of it.
|
|
59
|
+
*
|
|
60
|
+
* That near-unreachability was weighed and is ACCEPTED (#3477). Three loosenings were considered
|
|
61
|
+
* and all three are rejected:
|
|
62
|
+
*
|
|
63
|
+
* - Treat a primary-gateway 404 as authoritative absence. A gateway 404 means *that gateway did
|
|
64
|
+
* not resolve the CID inside its own timeout*, not that the content is absent from the
|
|
65
|
+
* network. Elevating it to a global absence claim is strictly weaker evidence than the current
|
|
66
|
+
* rule, in the exact direction #2647's FAILURE IS NOT ABSENCE exists to forbid.
|
|
67
|
+
* - Surface per-gateway evidence to the caller. Nothing to build: {@link
|
|
68
|
+
* IpfsFetchFailedError.causes} is already a per-candidate array in attempt order, and the
|
|
69
|
+
* aggregate message names each candidate. So the strictness costs no INFORMATION — only the
|
|
70
|
+
* convenience of a single verdict, which any caller needing finer granularity can bypass.
|
|
71
|
+
* - Separate the never-reached fallback out as its own case, distinct from a fallback that
|
|
72
|
+
* answered something other than 404. A candidate the whole-operation deadline never reached
|
|
73
|
+
* is not evidence of absence — the
|
|
74
|
+
* un-attempted gateway could have served it. Counting a non-attempt as absence-supporting is
|
|
75
|
+
* the first loosening with an extra step.
|
|
76
|
+
*
|
|
77
|
+
* The asymmetry is what settles it. Classifying genuine absence as `'unavailable'` costs one
|
|
78
|
+
* warning line and a retry on a later tick: cheap and self-correcting. Classifying a mere
|
|
79
|
+
* non-answer as `'not-found'` costs silent treatment as absent, no retry signal, and a data gap
|
|
80
|
+
* indistinguishable from real data: expensive, permanent, invisible.
|
|
81
|
+
*
|
|
82
|
+
* The strictness is not free: control flow does branch on `'not-found'`, so its cost is noisier
|
|
83
|
+
* `'unavailable'` warnings for content that is genuinely absent, rather than none —
|
|
84
|
+
* `buildFetchIpfsBytes` (returns `undefined` instead of the refusal string) and `narrowIpfsBytes`
|
|
85
|
+
* (silent instead of `logger.warn`) in `operator/src/daemon/composition-root.ts`;
|
|
86
|
+
* `warnIpfsFallThrough` (silent instead of `console.warn`) and `ipfsReason` (reason `not_found`
|
|
87
|
+
* instead of `too_large` / `unavailable`) in `./artifact-retrieval.ts`; and the task, trajectory,
|
|
88
|
+
* and source-bundle reads in `operator/src/conformance/harness.ts` (silent instead of
|
|
89
|
+
* `console.warn`).
|
|
59
90
|
*/
|
|
60
91
|
export declare function classifyIpfsFetchFailure(error: unknown): 'too-large' | 'not-found' | 'unavailable';
|
|
61
92
|
export declare function normalizeIpfsGatewayBase(gatewayUrl: string): string;
|
|
@@ -74,7 +105,14 @@ export type FetchFromIpfsOptions = {
|
|
|
74
105
|
* {@link DEFAULT_MAX_IPFS_RESPONSE_BYTES}. Raise it only for a call site that
|
|
75
106
|
* legitimately reads larger payloads (#3441) — the default is sized for JSON
|
|
76
107
|
* envelopes and source files, and a caller that raises it accepts buffering
|
|
77
|
-
* that many bytes. Values below 1 fall back to the default.
|
|
108
|
+
* that many bytes. Values below 1 fall back to the env, then the default.
|
|
109
|
+
*
|
|
110
|
+
* An operator may move the *default* with `JINN_IPFS_MAX_RESPONSE_BYTES`
|
|
111
|
+
* (#3453). An explicit bound here wins over it, so the env cannot clamp a
|
|
112
|
+
* call site that already states its own — concretely, it does not reach
|
|
113
|
+
* `fetchTrajectoryFromIpfs`'s 64 MiB (`MAX_TRAJECTORY_IPFS_RESPONSE_BYTES`,
|
|
114
|
+
* `operator/src/adapters/mech/ipfs.ts`). Making it a ceiling instead would
|
|
115
|
+
* make the raise-the-cap intent this option exists for inexpressible.
|
|
78
116
|
*/
|
|
79
117
|
maxResponseBytes?: number;
|
|
80
118
|
};
|
package/dist/corpus-read/ipfs.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { envInteger } from './env.js';
|
|
1
2
|
const IPFS_FETCH_TIMEOUT_MS = 15_000;
|
|
2
3
|
/**
|
|
3
4
|
* Bound on one whole `fetchFromIpfs` / `fetchBytesFromIpfs` call. The
|
|
@@ -79,6 +80,37 @@ export class IpfsFetchFailedError extends Error {
|
|
|
79
80
|
* default fetch also tries the `ipfs.io` fallback, which typically stalls into a 504 or an abort
|
|
80
81
|
* for an unpinned digest rather than answering 404. Such a run classifies `'unavailable'` — the
|
|
81
82
|
* safe direction, since absence is never claimed without proof of it.
|
|
83
|
+
*
|
|
84
|
+
* That near-unreachability was weighed and is ACCEPTED (#3477). Three loosenings were considered
|
|
85
|
+
* and all three are rejected:
|
|
86
|
+
*
|
|
87
|
+
* - Treat a primary-gateway 404 as authoritative absence. A gateway 404 means *that gateway did
|
|
88
|
+
* not resolve the CID inside its own timeout*, not that the content is absent from the
|
|
89
|
+
* network. Elevating it to a global absence claim is strictly weaker evidence than the current
|
|
90
|
+
* rule, in the exact direction #2647's FAILURE IS NOT ABSENCE exists to forbid.
|
|
91
|
+
* - Surface per-gateway evidence to the caller. Nothing to build: {@link
|
|
92
|
+
* IpfsFetchFailedError.causes} is already a per-candidate array in attempt order, and the
|
|
93
|
+
* aggregate message names each candidate. So the strictness costs no INFORMATION — only the
|
|
94
|
+
* convenience of a single verdict, which any caller needing finer granularity can bypass.
|
|
95
|
+
* - Separate the never-reached fallback out as its own case, distinct from a fallback that
|
|
96
|
+
* answered something other than 404. A candidate the whole-operation deadline never reached
|
|
97
|
+
* is not evidence of absence — the
|
|
98
|
+
* un-attempted gateway could have served it. Counting a non-attempt as absence-supporting is
|
|
99
|
+
* the first loosening with an extra step.
|
|
100
|
+
*
|
|
101
|
+
* The asymmetry is what settles it. Classifying genuine absence as `'unavailable'` costs one
|
|
102
|
+
* warning line and a retry on a later tick: cheap and self-correcting. Classifying a mere
|
|
103
|
+
* non-answer as `'not-found'` costs silent treatment as absent, no retry signal, and a data gap
|
|
104
|
+
* indistinguishable from real data: expensive, permanent, invisible.
|
|
105
|
+
*
|
|
106
|
+
* The strictness is not free: control flow does branch on `'not-found'`, so its cost is noisier
|
|
107
|
+
* `'unavailable'` warnings for content that is genuinely absent, rather than none —
|
|
108
|
+
* `buildFetchIpfsBytes` (returns `undefined` instead of the refusal string) and `narrowIpfsBytes`
|
|
109
|
+
* (silent instead of `logger.warn`) in `operator/src/daemon/composition-root.ts`;
|
|
110
|
+
* `warnIpfsFallThrough` (silent instead of `console.warn`) and `ipfsReason` (reason `not_found`
|
|
111
|
+
* instead of `too_large` / `unavailable`) in `./artifact-retrieval.ts`; and the task, trajectory,
|
|
112
|
+
* and source-bundle reads in `operator/src/conformance/harness.ts` (silent instead of
|
|
113
|
+
* `console.warn`).
|
|
82
114
|
*/
|
|
83
115
|
export function classifyIpfsFetchFailure(error) {
|
|
84
116
|
const causes = error instanceof IpfsFetchFailedError ? error.causes : [error];
|
|
@@ -93,20 +125,32 @@ export function normalizeIpfsGatewayBase(gatewayUrl) {
|
|
|
93
125
|
let normalized = gatewayUrl.trim();
|
|
94
126
|
if (normalized === '')
|
|
95
127
|
normalized = 'https://gateway.autonolas.tech';
|
|
96
|
-
// Drop any userinfo at the source. `fetch` rejects a credentialed URL
|
|
97
|
-
// outright, and its own error message quotes the URL back — so a gateway
|
|
98
|
-
// configured Infura-style would otherwise put its secret into every
|
|
99
|
-
// aggregated fetch error, which callers log.
|
|
100
128
|
try {
|
|
101
129
|
const parsed = new URL(normalized);
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
130
|
+
// Drop any userinfo at the source. `fetch` rejects a credentialed URL
|
|
131
|
+
// outright, and its own error message quotes the URL back — so a gateway
|
|
132
|
+
// configured Infura-style would otherwise put its secret into every
|
|
133
|
+
// aggregated fetch error, which callers log.
|
|
134
|
+
parsed.username = '';
|
|
135
|
+
parsed.password = '';
|
|
136
|
+
// A fragment is never transmitted to a server, and treating it as part of
|
|
137
|
+
// the base corrupted the `/ipfs` suffix test into appending a second
|
|
138
|
+
// segment (`…/ipfs#frag` → `…/ipfs#frag/ipfs/`). Dropping it is the only
|
|
139
|
+
// reading that is both correct and lossless in transit.
|
|
140
|
+
parsed.hash = '';
|
|
141
|
+
// Normalize on the parsed path, not the raw string: a query-bearing base
|
|
142
|
+
// otherwise had `/ipfs/` spliced on after the query, putting the CID
|
|
143
|
+
// inside the query string (#3452).
|
|
144
|
+
let path = parsed.pathname.replace(/\/+$/, '');
|
|
145
|
+
if (!path.toLowerCase().endsWith('/ipfs'))
|
|
146
|
+
path = `${path}/ipfs`;
|
|
147
|
+
parsed.pathname = `${path}/`;
|
|
148
|
+
return parsed.toString();
|
|
107
149
|
}
|
|
108
150
|
catch {
|
|
109
|
-
// Not an absolute URL; leave it to the caller's own failure path
|
|
151
|
+
// Not an absolute URL; leave it to the caller's own failure path, which
|
|
152
|
+
// reports it as a per-candidate `candidate URL could not be parsed`
|
|
153
|
+
// instead of escaping as a bare TypeError.
|
|
110
154
|
}
|
|
111
155
|
normalized = normalized.replace(/\/+$/, '');
|
|
112
156
|
if (!normalized.toLowerCase().endsWith('/ipfs'))
|
|
@@ -190,7 +234,51 @@ async function discardBody(response) {
|
|
|
190
234
|
// A body that refuses to cancel is not worth failing the request over.
|
|
191
235
|
}
|
|
192
236
|
}
|
|
193
|
-
/**
|
|
237
|
+
/**
|
|
238
|
+
* The exact decoded length this response promises, or `undefined` when it
|
|
239
|
+
* promises nothing usable (#3759).
|
|
240
|
+
*
|
|
241
|
+
* `content-encoding` is the trap: with one set, `content-length` describes
|
|
242
|
+
* *wire* bytes, and the reader hands back the decoded ones — a gzipped 200 KB
|
|
243
|
+
* JSON body declares ~230. Pre-sizing from that is not a pessimization, it is
|
|
244
|
+
* a buffer guaranteed to overflow, so the fast path is gated on the encoding
|
|
245
|
+
* being absent or `identity`. The raw header is read rather than
|
|
246
|
+
* `Number(header)` because `Number(null)` is `0`, which cannot be told from a
|
|
247
|
+
* genuinely declared zero.
|
|
248
|
+
*/
|
|
249
|
+
function exactDecodedLength(response, maxBytes) {
|
|
250
|
+
const encoding = response.headers.get('content-encoding');
|
|
251
|
+
if (encoding !== null && encoding.trim().toLowerCase() !== 'identity')
|
|
252
|
+
return undefined;
|
|
253
|
+
const raw = response.headers.get('content-length');
|
|
254
|
+
if (raw === null || raw.trim() === '')
|
|
255
|
+
return undefined;
|
|
256
|
+
const declared = Number(raw);
|
|
257
|
+
if (!Number.isInteger(declared) || declared < 0 || declared > maxBytes)
|
|
258
|
+
return undefined;
|
|
259
|
+
return declared;
|
|
260
|
+
}
|
|
261
|
+
/**
|
|
262
|
+
* Read a response body as bytes, refusing anything past `maxBytes`.
|
|
263
|
+
*
|
|
264
|
+
* Peak resident bytes, not memcpy count, is what the pre-sizing below buys.
|
|
265
|
+
* The stream has already allocated each chunk either way, and the join is the
|
|
266
|
+
* single copy in both designs — but the chunk-list path holds every chunk live
|
|
267
|
+
* *while* it allocates and fills the joined buffer, so it peaks at roughly
|
|
268
|
+
* twice the payload: ~128 MiB at `fetchTrajectoryFromIpfs`'s 64 MiB bound
|
|
269
|
+
* (measured), against ~1x live for the pre-sized path.
|
|
270
|
+
*
|
|
271
|
+
* The honest caveat: the fast path is gated on the absence of a content
|
|
272
|
+
* encoding, and gateways generally compress JSON — so it will rarely engage on
|
|
273
|
+
* the very trajectory read that motivated the bound. The win is real for the
|
|
274
|
+
* raw-bytes path (source files, sealed documents, envelopes) and
|
|
275
|
+
* correct-but-inert for compressed JSON.
|
|
276
|
+
*
|
|
277
|
+
* Out of scope, and larger than what this removes: `fetchJson` decodes these
|
|
278
|
+
* bytes into a UTF-16 string (up to 2x the byte length) and then `JSON.parse`s
|
|
279
|
+
* that into an object graph, all while the bytes are still live. Bringing that
|
|
280
|
+
* down means streaming JSON parsing, which is a rewrite rather than a fix.
|
|
281
|
+
*/
|
|
194
282
|
async function readBoundedBytes(response, maxBytes) {
|
|
195
283
|
const declared = Number(response.headers.get('content-length'));
|
|
196
284
|
if (Number.isFinite(declared) && declared > maxBytes) {
|
|
@@ -201,6 +289,9 @@ async function readBoundedBytes(response, maxBytes) {
|
|
|
201
289
|
if (!body)
|
|
202
290
|
return new Uint8Array(0);
|
|
203
291
|
const reader = body.getReader();
|
|
292
|
+
const exact = exactDecodedLength(response, maxBytes);
|
|
293
|
+
let buffer = exact === undefined ? undefined : new Uint8Array(exact);
|
|
294
|
+
let written = 0;
|
|
204
295
|
const chunks = [];
|
|
205
296
|
let total = 0;
|
|
206
297
|
try {
|
|
@@ -211,9 +302,27 @@ async function readBoundedBytes(response, maxBytes) {
|
|
|
211
302
|
if (!value)
|
|
212
303
|
continue;
|
|
213
304
|
total += value.byteLength;
|
|
305
|
+
// The cumulative counter stays the authority on the cap. The pre-sized
|
|
306
|
+
// buffer's own length is never it — a lying `content-length` must not be
|
|
307
|
+
// able to widen or narrow the bound.
|
|
214
308
|
if (total > maxBytes) {
|
|
215
309
|
throw new IpfsResponseTooLargeError(maxBytes);
|
|
216
310
|
}
|
|
311
|
+
if (buffer !== undefined) {
|
|
312
|
+
if (written + value.byteLength <= buffer.length) {
|
|
313
|
+
buffer.set(value, written);
|
|
314
|
+
written += value.byteLength;
|
|
315
|
+
continue;
|
|
316
|
+
}
|
|
317
|
+
// The body outran what it declared, on an identity response. Fall back
|
|
318
|
+
// in place: what was written becomes the first chunk-list entry and
|
|
319
|
+
// the read continues exactly as it did before. Peak on this anomalous
|
|
320
|
+
// path is 2x — today's behaviour, so no regression and no new failure
|
|
321
|
+
// mode. (`subarray` is right here: the pre-sized buffer is dropped
|
|
322
|
+
// straight after, so retaining its backing store costs nothing.)
|
|
323
|
+
chunks.push(buffer.subarray(0, written));
|
|
324
|
+
buffer = undefined;
|
|
325
|
+
}
|
|
217
326
|
chunks.push(value);
|
|
218
327
|
}
|
|
219
328
|
}
|
|
@@ -225,6 +334,15 @@ async function readBoundedBytes(response, maxBytes) {
|
|
|
225
334
|
// Already terminal; the read result (or throw) above is what matters.
|
|
226
335
|
}
|
|
227
336
|
}
|
|
337
|
+
if (buffer !== undefined) {
|
|
338
|
+
// A short body is not reachable through `fetch` — undici rejects a body
|
|
339
|
+
// that ends early with `TypeError: terminated` — but returning trailing
|
|
340
|
+
// zeros would be wrong if it ever were, and the guard costs one branch.
|
|
341
|
+
// `slice`, never `subarray`: a view would retain the whole pre-sized
|
|
342
|
+
// backing store and silently defeat this change on the one path where it
|
|
343
|
+
// matters.
|
|
344
|
+
return written === buffer.length ? buffer : buffer.slice(0, written);
|
|
345
|
+
}
|
|
228
346
|
const joined = new Uint8Array(total);
|
|
229
347
|
let offset = 0;
|
|
230
348
|
for (const chunk of chunks) {
|
|
@@ -303,10 +421,14 @@ async function fetchBytes(url, signal, maxBytes) {
|
|
|
303
421
|
}
|
|
304
422
|
function resolveMaxResponseBytes(opts) {
|
|
305
423
|
const requested = opts?.maxResponseBytes;
|
|
306
|
-
if (typeof requested
|
|
307
|
-
return
|
|
308
|
-
}
|
|
309
|
-
|
|
424
|
+
if (typeof requested === 'number' && Number.isFinite(requested) && requested >= 1) {
|
|
425
|
+
return Math.floor(requested);
|
|
426
|
+
}
|
|
427
|
+
// The env replaces the default as the fallback; it is not a second
|
|
428
|
+
// validation layer, so an out-of-range *option* still falls through to it.
|
|
429
|
+
// `minimum = 1` is what keeps the cap from being disabled into an unbounded
|
|
430
|
+
// read: `0` and every other out-of-range value land on the default.
|
|
431
|
+
return envInteger('JINN_IPFS_MAX_RESPONSE_BYTES', DEFAULT_MAX_IPFS_RESPONSE_BYTES, 1);
|
|
310
432
|
}
|
|
311
433
|
function resolveFallbackGatewayBases(opts) {
|
|
312
434
|
if (opts?.fallbackGatewayBase === false)
|
|
@@ -347,6 +469,24 @@ function resolveGatewayCandidateUrl(baseUrl, cidPath) {
|
|
|
347
469
|
if (resolved.origin !== base.origin || !resolved.pathname.startsWith(base.pathname)) {
|
|
348
470
|
return { reason: 'candidate URL escapes the gateway path prefix' };
|
|
349
471
|
}
|
|
472
|
+
// `new URL(cidPath, base)` drops the base's query unconditionally, which
|
|
473
|
+
// strips the API key off every request to an authenticated gateway (#3452).
|
|
474
|
+
// Re-attach it *after* the guard, so a manifest-supplied CID path can never
|
|
475
|
+
// influence the check, and only when the base has one, so every query-free
|
|
476
|
+
// base issues a byte-identical request. The base wins over any query the CID
|
|
477
|
+
// path carries: two query strings cannot be merged unambiguously, and
|
|
478
|
+
// letting manifest text override an operator's gateway credentials is the
|
|
479
|
+
// wrong direction.
|
|
480
|
+
//
|
|
481
|
+
// Two known limits, both narrow and neither fixed here. A *relative*
|
|
482
|
+
// redirect `Location` is resolved with `new URL(location, current)`, which
|
|
483
|
+
// drops the query, so the key is not forwarded across such a hop — gateways
|
|
484
|
+
// that redirect path form to subdomain form send an absolute `Location`, and
|
|
485
|
+
// `assertRedirectAllowed` already pins the host family and port. And
|
|
486
|
+
// `displayUrl` stays `origin + pathname`, which is what keeps the credential
|
|
487
|
+
// out of logs now that the request actually carries it.
|
|
488
|
+
if (base.search !== '')
|
|
489
|
+
resolved.search = base.search;
|
|
350
490
|
return { url: resolved };
|
|
351
491
|
}
|
|
352
492
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@jinn-network/core",
|
|
3
|
-
"version": "0.1.2-canary.
|
|
3
|
+
"version": "0.1.2-canary.c8da24a8",
|
|
4
4
|
"description": "Domain core for Jinn — evidence/contribution stores, scrub, trajectory parsing, and corpus reads. Independent of operator/src.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"packageManager": "yarn@4.13.0",
|
|
@@ -43,7 +43,7 @@
|
|
|
43
43
|
},
|
|
44
44
|
"dependencies": {
|
|
45
45
|
"@huggingface/transformers": "^4.2.0",
|
|
46
|
-
"@jinn-network/plugin": "0.1.2-canary.
|
|
46
|
+
"@jinn-network/plugin": "0.1.2-canary.c8da24a8",
|
|
47
47
|
"@lmoe/gliner-onnx": "0.1.0",
|
|
48
48
|
"@noble/hashes": "^2.2.0",
|
|
49
49
|
"@secretlint/core": "^13.0.2",
|