@jinn-network/core 0.1.2-canary.bebeb033 → 0.1.2-canary.c8da24a8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,6 +3,12 @@
3
3
  *
4
4
  * Always hash-verifies before persisting to cache.
5
5
  *
6
+ * The two *network* legs — the donated IPFS mirror and the operator origin —
7
+ * are not implemented here: they are `fetchVerifiedArtifact`, the keyless
8
+ * retrieval primitive (#4179). What remains in this file is the store-aware
9
+ * composition around it: the cache read, the self-store mirror, the route
10
+ * resolver, and the cache writes whose ordering the daemon owns.
11
+ *
6
12
  * Spec §2.3 step 4-6.
7
13
  */
8
14
  import { type AcquireResult } from './fetch-artifact.js';
@@ -3,72 +3,68 @@
3
3
  *
4
4
  * Always hash-verifies before persisting to cache.
5
5
  *
6
+ * The two *network* legs — the donated IPFS mirror and the operator origin —
7
+ * are not implemented here: they are `fetchVerifiedArtifact`, the keyless
8
+ * retrieval primitive (#4179). What remains in this file is the store-aware
9
+ * composition around it: the cache read, the self-store mirror, the route
10
+ * resolver, and the cache writes whose ordering the daemon owns.
11
+ *
6
12
  * Spec §2.3 step 4-6.
7
13
  */
8
- import { createHash } from 'node:crypto';
9
14
  import { fetchArtifactContent } from './fetch-artifact.js';
15
+ import { fetchVerifiedArtifact, verifyArtifactDigest } from './artifact-retrieval.js';
10
16
  import { AcquireError, HashMismatchError } from './types.js';
11
- import { classifyIpfsFetchFailure, fetchFromIpfs as defaultFetchFromIpfs } from './ipfs.js';
12
- const DONATION_ARTIFACT_ENCODING = 'jinn.artifact.donation.v1';
13
- function sha256Hex(buf) {
14
- return createHash('sha256').update(buf).digest('hex');
15
- }
16
17
  /**
17
- * Log an IPFS read that the acquisition chain is about to fall through, unless
18
- * the gateway actually answered "not there" (#3441). A cap refusal is positive
19
- * evidence the content exists, so it must not read like an ordinary miss.
18
+ * Adapt a legacy `AcquireFn` to the primitive's clean seam. The legacy shape —
19
+ * a third `privateKey` parameter and a `Buffer | null` return — exists for
20
+ * older test fakes and stays confined to this file.
20
21
  */
21
- function warnIpfsFallThrough(subject, error) {
22
- const classification = classifyIpfsFetchFailure(error);
23
- if (classification === 'not-found')
24
- return;
25
- const detail = error instanceof Error ? error.message : String(error);
26
- console.warn(`[corpus-read] IPFS source for ${subject} could not be used (${classification}), `
27
- + `falling through to the next source: ${detail}`);
28
- }
29
- function decodeDonationArtifact(raw, expectedSha256) {
30
- if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
31
- throw new Error('donation artifact payload is not an object');
32
- }
33
- const record = raw;
34
- if (record['schemaVersion'] !== DONATION_ARTIFACT_ENCODING || record['encoding'] !== DONATION_ARTIFACT_ENCODING) {
35
- throw new Error('donation artifact payload has unexpected encoding');
36
- }
37
- if (record['sha256'] !== expectedSha256) {
38
- throw new Error('donation artifact sha256 does not match requested artifact');
39
- }
40
- if (typeof record['data'] !== 'string') {
41
- throw new Error('donation artifact payload is missing base64 data');
42
- }
43
- return Buffer.from(record['data'], 'base64');
22
+ function toAcquireResultFn(acquireFn, privateKey) {
23
+ return async (endpoint, sha256) => {
24
+ const raw = await acquireFn(endpoint, sha256, privateKey);
25
+ if (raw === null)
26
+ return { ok: false, reason: 'not_found', message: 'origin returned null (404 / not found)' };
27
+ if (Buffer.isBuffer(raw))
28
+ return { ok: true, content: raw };
29
+ return raw;
30
+ };
44
31
  }
45
32
  export async function acquireArtifactContent(args) {
46
- const { sha256, artifactType, access, store, selfSafeAddress, privateKey, routeResolver, envelopeCid, sources = [], ipfsGatewayUrl, ownerSafe, acquireFn = (endpoint, sha256) => fetchArtifactContent(endpoint, sha256), fetchFromIpfs = defaultFetchFromIpfs, } = args;
33
+ const { sha256, artifactType, access, store, selfSafeAddress, privateKey, routeResolver, envelopeCid, sources = [], ipfsGatewayUrl, ownerSafe, acquireFn = (endpoint, sha256) => fetchArtifactContent(endpoint, sha256), fetchFromIpfs, } = args;
47
34
  const now = () => new Date().toISOString();
48
- // 1. Cache hit
35
+ // 1. Cache hit. network_artifacts is a cache, not authoritative local state:
36
+ // a row that does not hash to its own key is a MISS, and every leg below
37
+ // re-saves through INSERT OR REPLACE, so falling through repairs it. (The
38
+ // self-store path at step 3 throws instead, and rightly — there is no
39
+ // upstream to re-fetch a corrupt served_artifacts row from.)
49
40
  const cached = store.getNetworkArtifact(sha256);
50
41
  if (cached) {
51
- store.touchNetworkArtifactUsage(sha256, now());
52
- return {
53
- sha256,
54
- bytes: cached.content,
55
- artifactType: cached.artifactType,
56
- source: 'cache',
57
- paidAmountUsdc: '0',
58
- fetchedAt: cached.fetchedAt,
59
- sourceOperator: cached.sourceOperator ?? undefined,
60
- };
42
+ // Verified before the usage bump: a row we are about to discard must not
43
+ // have its last_used_at refreshed.
44
+ const verifiedCache = verifyArtifactDigest(sha256, cached.content);
45
+ if (verifiedCache.ok) {
46
+ store.touchNetworkArtifactUsage(sha256, now());
47
+ return {
48
+ sha256,
49
+ bytes: cached.content,
50
+ artifactType: cached.artifactType,
51
+ source: 'cache',
52
+ paidAmountUsdc: '0',
53
+ fetchedAt: cached.fetchedAt,
54
+ sourceOperator: cached.sourceOperator ?? undefined,
55
+ };
56
+ }
57
+ console.warn(`[corpus-read] cached artifact ${sha256} hashed to ${verifiedCache.actualSha256}; `
58
+ + 'treating the row as a miss and re-fetching');
61
59
  }
62
- // 2. Public IPFS donation source.
60
+ // 2. Public IPFS donation source. The primitive owns the fetch, the decode,
61
+ // the verification, and the #3441 fall-through warning; the cache write and
62
+ // the mismatch-is-fatal decision stay here, where the store lives.
63
63
  const ipfsSource = sources.find((source) => source.kind === 'ipfs');
64
64
  if (ipfsSource && ipfsGatewayUrl) {
65
- try {
66
- const raw = await fetchFromIpfs(ipfsGatewayUrl, ipfsSource.cid);
67
- const bytes = decodeDonationArtifact(raw, sha256);
68
- const actualSha = sha256Hex(bytes);
69
- if (actualSha !== sha256) {
70
- throw new HashMismatchError(sha256, actualSha, 'ipfs', ownerSafe);
71
- }
65
+ const retrieved = await fetchVerifiedArtifact({ sha256, artifactType }, { sources, ipfsGatewayUrl, ownerSafe, envelopeCid }, { deps: { fetchFromIpfs } });
66
+ if (retrieved.ok) {
67
+ const bytes = retrieved.artifact.bytes;
72
68
  const ts = now();
73
69
  store.saveNetworkArtifact({
74
70
  sha256,
@@ -91,18 +87,13 @@ export async function acquireArtifactContent(args) {
91
87
  sourceOperator: ownerSafe,
92
88
  };
93
89
  }
94
- catch (err) {
95
- if (err instanceof HashMismatchError)
96
- throw err;
97
- // Donated IPFS is an opportunistic fast path. Gateway failures and
98
- // malformed donation payloads should not block the acquisition chain;
99
- // only verified byte hash mismatches remain fatal.
100
- //
101
- // Swallowing the failure silently, though, turns a size-policy refusal or
102
- // a blocked redirect into an invisible data gap (#3441). Classify before
103
- // swallowing so anything that is not proven absence reaches the operator.
104
- warnIpfsFallThrough(`donation artifact ${sha256}`, err);
90
+ if (retrieved.reason === 'digest_mismatch') {
91
+ throw new HashMismatchError(sha256, retrieved.mismatch.actualSha256, 'ipfs', ownerSafe);
105
92
  }
93
+ // Donated IPFS is an opportunistic fast path: gateway failures and
94
+ // malformed donation payloads fall through to the next source. Anything
95
+ // that was not proven absence has already reached the operator as a
96
+ // warning from inside the primitive (#3441).
106
97
  }
107
98
  // 3. Self-store fast path
108
99
  if (ownerSafe && ownerSafe.toLowerCase() === selfSafeAddress.toLowerCase()) {
@@ -113,9 +104,9 @@ export async function acquireArtifactContent(args) {
113
104
  // propagate the bad bytes into network_artifacts where peers can
114
105
  // fetch them. Throwing closes the cache-poisoning gap; the self-store
115
106
  // path is now hash-equivalent to origin / route-resolver.
116
- const actualSha = sha256Hex(own.content);
117
- if (actualSha !== sha256) {
118
- throw new HashMismatchError(sha256, actualSha, 'self-store', selfSafeAddress);
107
+ const verified = verifyArtifactDigest(sha256, own.content);
108
+ if (!verified.ok) {
109
+ throw new HashMismatchError(sha256, verified.actualSha256, 'self-store', selfSafeAddress);
119
110
  }
120
111
  // Mirror into cache so peer asks for the same content can hit cache (provenance: self-store-mirror).
121
112
  const ts = now();
@@ -143,9 +134,9 @@ export async function acquireArtifactContent(args) {
143
134
  try {
144
135
  const out = await routeResolver.resolve({ sha256, access, requesterSafe: selfSafeAddress });
145
136
  if (out) {
146
- const actualSha = sha256Hex(out.bytes);
147
- if (actualSha !== sha256) {
148
- throw new HashMismatchError(sha256, actualSha, 'route-resolver', out.sourceOperator);
137
+ const verified = verifyArtifactDigest(sha256, out.bytes);
138
+ if (!verified.ok) {
139
+ throw new HashMismatchError(sha256, verified.actualSha256, 'route-resolver', out.sourceOperator);
149
140
  }
150
141
  const ts = now();
151
142
  store.saveNetworkArtifact({
@@ -176,36 +167,17 @@ export async function acquireArtifactContent(args) {
176
167
  }
177
168
  }
178
169
  // 5. Origin fetch
179
- let bytes;
180
- try {
181
- const raw = await acquireFn(access.endpoint, sha256, privateKey);
182
- // Accept both the legacy `Buffer | null` shape and the discriminated
183
- // union from fetchArtifactContent. Buffer.isBuffer guards before we
184
- // duck-type into the union.
185
- if (raw === null || Buffer.isBuffer(raw)) {
186
- bytes = raw;
170
+ const retrieved = await fetchVerifiedArtifact({ sha256, artifactType }, { endpoint: access.endpoint, ownerSafe, envelopeCid }, { deps: { fetchArtifact: toAcquireResultFn(acquireFn, privateKey) } });
171
+ if (!retrieved.ok) {
172
+ if (retrieved.reason === 'digest_mismatch') {
173
+ throw new HashMismatchError(sha256, retrieved.mismatch.actualSha256, 'origin', ownerSafe);
187
174
  }
188
- else if (raw.ok === true) {
189
- bytes = raw.content;
190
- }
191
- else {
192
- // ok: false — preserve the reason so the caller / daemon route can
193
- // surface it via AcquireError.cause.
194
- throw new AcquireError(sha256, `origin fetch failed: ${raw.reason}${raw.message ? ` (${raw.message})` : ''}`, raw);
195
- }
196
- }
197
- catch (err) {
198
- if (err instanceof AcquireError)
199
- throw err;
200
- throw new AcquireError(sha256, 'origin fetch failed', err);
201
- }
202
- if (!bytes) {
203
- throw new AcquireError(sha256, 'origin returned null (404 / not found)');
204
- }
205
- const actualSha = sha256Hex(bytes);
206
- if (actualSha !== sha256) {
207
- throw new HashMismatchError(sha256, actualSha, 'origin', ownerSafe);
175
+ // Preserve the reason token in the message so callers and the daemon route
176
+ // can surface which failure this was.
177
+ const attempt = retrieved.attempts.find((entry) => entry.leg === 'origin');
178
+ throw new AcquireError(sha256, `origin fetch failed: ${retrieved.reason}${attempt?.message ? ` (${attempt.message})` : ''}`, retrieved);
208
179
  }
180
+ const bytes = retrieved.artifact.bytes;
209
181
  const ts = now();
210
182
  store.saveNetworkArtifact({
211
183
  sha256,
@@ -0,0 +1,181 @@
1
+ /**
2
+ * Artifact retrieval primitive — fetch by content address, verify, return bytes.
3
+ *
4
+ * One function fetches an artifact from the network legs an envelope names and
5
+ * hands back its bytes only after they hash to the recorded sha256, together
6
+ * with the provenance of the leg that produced them. A digest mismatch fails
7
+ * closed: the failure arm of the result union has no bytes on it at all.
8
+ *
9
+ * Keyless and filesystem-neutral by construction. The primitive itself neither
10
+ * accepts nor touches key material or the filesystem: it takes no store, no
11
+ * signer, no private key, and no path, and it performs no write of any kind.
12
+ * The one channel through which such material can reach a call is the caller's
13
+ * own `deps` seam closure — `acquire.ts` legitimately supplies one that holds a
14
+ * private key — and that closure is opaque here: it is invoked, never read,
15
+ * never logged, and never surfaced in provenance.
16
+ * `test/architecture/keyless-primitive.test.ts` holds the module graph to the
17
+ * same promise.
18
+ *
19
+ * Where the returned bytes land is the caller's decision. The daemon chain
20
+ * (`acquire.ts`) composes this with its byte cache; the CLI hands them to the
21
+ * operator; the MCP handler proxies to a daemon that runs it. None of that is
22
+ * this function's business.
23
+ *
24
+ * Issue #4179 (umbrella #2448 / journey spec B0c).
25
+ */
26
+ import { type AcquireResult } from './fetch-artifact.js';
27
+ import type { ArtifactSource } from './types.js';
28
+ /**
29
+ * Content address of a corpus artifact: the sha256 of the bytes themselves, as
30
+ * recorded in the envelope's `artifact.sha256` — never an IPFS CID (artifacts
31
+ * stopped having CIDs post jinn-mono-vy37.1.2).
32
+ */
33
+ export interface ArtifactAddress {
34
+ /** 64-char lowercase hex. The one thing that is trusted. */
35
+ sha256: string;
36
+ /**
37
+ * Declared type from the envelope. Carried into the result for the caller's
38
+ * convenience and never consulted during verification — an attacker-supplied
39
+ * label cannot influence whether bytes are admitted.
40
+ */
41
+ artifactType?: string;
42
+ }
43
+ /**
44
+ * Where the bytes might be. Every field is manifest-supplied and therefore
45
+ * attacker-controlled; the destination policy and the byte/redirect/deadline
46
+ * caps in `fetch-artifact.ts` and `ipfs.ts` are what make that safe.
47
+ */
48
+ export interface ArtifactLocators {
49
+ /** Donated IPFS sources — the envelope's `artifact.sources` entries. */
50
+ sources?: readonly ArtifactSource[];
51
+ /** Gateway base for the IPFS leg. The leg is skipped when this is absent. */
52
+ ipfsGatewayUrl?: string;
53
+ /** HTTP origin — the envelope's `artifact.access.endpoint`. */
54
+ endpoint?: string;
55
+ /** Envelope CID this artifact was named by. Provenance only. */
56
+ envelopeCid?: string;
57
+ /** Publishing operator's Safe, when known. Provenance and attribution only. */
58
+ ownerSafe?: string;
59
+ }
60
+ export interface FetchVerifiedArtifactOptions {
61
+ /**
62
+ * Injection seams, mirroring the `acquire.ts` precedent so tests need no
63
+ * module mock. A caller that needs to bound or redirect a leg replaces the
64
+ * leg — there is no separate options passthrough, because no call site wants
65
+ * the default transport with different bounds.
66
+ */
67
+ deps?: {
68
+ fetchArtifact?: (endpoint: string, sha256: string) => Promise<AcquireResult>;
69
+ fetchFromIpfs?: (gatewayUrl: string, cid: string) => Promise<unknown>;
70
+ /** Clock seam for `fetchedAt`. */
71
+ now?: () => string;
72
+ };
73
+ }
74
+ /** One leg's outcome, recorded whether it succeeded, was skipped, or failed. */
75
+ export interface ArtifactFetchAttempt {
76
+ readonly leg: 'ipfs' | 'origin';
77
+ /** `ipfs://<cid>` or the exact URL `buildArtifactUrl` produced; empty when skipped. */
78
+ readonly sourceUri: string;
79
+ readonly outcome: 'verified' | 'skipped' | 'failed' | 'digest_mismatch';
80
+ /**
81
+ * Present on `failed`; the classified reason, never a raw string match.
82
+ * `digest_mismatch` is excluded because `refuse()` records that outcome with
83
+ * no `reason` field — no attempt ever carries the token, and saying so here
84
+ * is what lets the failure union below narrow without a cast.
85
+ */
86
+ readonly reason?: Exclude<ArtifactFetchFailureReason, 'digest_mismatch'>;
87
+ readonly message?: string;
88
+ }
89
+ export interface ArtifactProvenance {
90
+ /** Which leg produced the admitted bytes. */
91
+ readonly source: 'ipfs' | 'origin';
92
+ readonly sourceUri: string;
93
+ /** Always 'sha256'. Explicit so the record does not silently mean "some digest". */
94
+ readonly digestAlgorithm: 'sha256';
95
+ /** ISO-8601 instant the bytes were verified, not when they were requested. */
96
+ readonly fetchedAt: string;
97
+ readonly sourceOperator?: string;
98
+ readonly envelopeCid?: string;
99
+ /** Every leg tried, in order. Legible even on the happy path. */
100
+ readonly attempts: readonly ArtifactFetchAttempt[];
101
+ }
102
+ export interface VerifiedArtifact {
103
+ readonly sha256: string;
104
+ /** Verified bytes, in memory. Never a path, never a handle. */
105
+ readonly bytes: Buffer;
106
+ readonly artifactType: string;
107
+ readonly provenance: ArtifactProvenance;
108
+ }
109
+ export type ArtifactFetchFailureReason =
110
+ /** Bytes arrived and did not hash to the address. */
111
+ 'digest_mismatch'
112
+ /** Every attempted leg answered, and every answer was absence. */
113
+ | 'not_found'
114
+ /** Nothing was supplied to try. */
115
+ | 'no_locator'
116
+ /** Destination policy refused an origin or a redirect hop. */
117
+ | 'blocked' | 'too_large' | 'timeout'
118
+ /** Transport failure — nothing was learned about whether the content exists. */
119
+ | 'unavailable'
120
+ /**
121
+ * A locator resolved and what it served is permanently not this artifact —
122
+ * a broken donation payload. Conclusive absence AT THAT LOCATOR, so it is
123
+ * not retryable; never conclusive globally, so it never outranks a leg that
124
+ * learned nothing (#3441).
125
+ */
126
+ | 'malformed_payload';
127
+ /** Evidence, not content. Present on `digest_mismatch` and only there. */
128
+ export interface ArtifactDigestMismatch {
129
+ readonly expectedSha256: string;
130
+ readonly actualSha256: string;
131
+ readonly sourceUri: string;
132
+ readonly sourceOperator?: string;
133
+ }
134
+ interface FetchVerifiedArtifactFailureBase {
135
+ readonly ok: false;
136
+ readonly sha256: string;
137
+ /** Derived from `reason`, never caller-supplied. */
138
+ readonly retryable: boolean;
139
+ readonly message: string;
140
+ readonly attempts: readonly ArtifactFetchAttempt[];
141
+ }
142
+ /**
143
+ * Discriminated on `reason` so the compiler, not a constructor comment, carries
144
+ * the invariant that `digest_mismatch` always ships its evidence. `refuse()` is
145
+ * the sole producer of that reason and has always set `mismatch`; before this
146
+ * split, callers reading the evidence had to assert it with `!`.
147
+ */
148
+ export type FetchVerifiedArtifactFailure = (FetchVerifiedArtifactFailureBase & {
149
+ readonly reason: 'digest_mismatch';
150
+ readonly mismatch: ArtifactDigestMismatch;
151
+ }) | (FetchVerifiedArtifactFailureBase & {
152
+ readonly reason: Exclude<ArtifactFetchFailureReason, 'digest_mismatch'>;
153
+ readonly mismatch?: undefined;
154
+ });
155
+ export type FetchVerifiedArtifactResult = {
156
+ readonly ok: true;
157
+ readonly artifact: VerifiedArtifact;
158
+ } | FetchVerifiedArtifactFailure;
159
+ /**
160
+ * The one digest check in the system. Legs that already hold bytes — the byte
161
+ * cache, the self-store, a route resolver, the MCP daemon proxy — call this
162
+ * rather than writing `createHash('sha256')` out again.
163
+ */
164
+ export declare function verifyArtifactDigest(expectedSha256: string, bytes: Buffer): {
165
+ ok: true;
166
+ } | {
167
+ ok: false;
168
+ actualSha256: string;
169
+ };
170
+ /**
171
+ * Fetch an artifact by content address and return its bytes only after they
172
+ * hash to that address.
173
+ *
174
+ * Legs run in the order the daemon chain has always used: the free donated IPFS
175
+ * mirror first, the operator's HTTP origin second. A leg with no locator is
176
+ * *skipped*, not failed. A digest mismatch on any leg fails the whole call and
177
+ * does not fall through — the bytes are wrong or hostile, and the next locator
178
+ * cannot make them right.
179
+ */
180
+ export declare function fetchVerifiedArtifact(address: ArtifactAddress, locators: ArtifactLocators, options?: FetchVerifiedArtifactOptions): Promise<FetchVerifiedArtifactResult>;
181
+ export {};
@@ -0,0 +1,268 @@
1
+ /**
2
+ * Artifact retrieval primitive — fetch by content address, verify, return bytes.
3
+ *
4
+ * One function fetches an artifact from the network legs an envelope names and
5
+ * hands back its bytes only after they hash to the recorded sha256, together
6
+ * with the provenance of the leg that produced them. A digest mismatch fails
7
+ * closed: the failure arm of the result union has no bytes on it at all.
8
+ *
9
+ * Keyless and filesystem-neutral by construction. The primitive itself neither
10
+ * accepts nor touches key material or the filesystem: it takes no store, no
11
+ * signer, no private key, and no path, and it performs no write of any kind.
12
+ * The one channel through which such material can reach a call is the caller's
13
+ * own `deps` seam closure — `acquire.ts` legitimately supplies one that holds a
14
+ * private key — and that closure is opaque here: it is invoked, never read,
15
+ * never logged, and never surfaced in provenance.
16
+ * `test/architecture/keyless-primitive.test.ts` holds the module graph to the
17
+ * same promise.
18
+ *
19
+ * Where the returned bytes land is the caller's decision. The daemon chain
20
+ * (`acquire.ts`) composes this with its byte cache; the CLI hands them to the
21
+ * operator; the MCP handler proxies to a daemon that runs it. None of that is
22
+ * this function's business.
23
+ *
24
+ * Issue #4179 (umbrella #2448 / journey spec B0c).
25
+ */
26
+ import { createHash } from 'node:crypto';
27
+ import { buildArtifactUrl, fetchArtifactContent, } from './fetch-artifact.js';
28
+ import { classifyIpfsFetchFailure, fetchFromIpfs as defaultFetchFromIpfs } from './ipfs.js';
29
+ const DONATION_ARTIFACT_ENCODING = 'jinn.artifact.donation.v1';
30
+ /** Nothing that was learned tells us to stop; everything else does. */
31
+ const RETRYABLE_REASONS = new Set([
32
+ 'timeout',
33
+ 'unavailable',
34
+ ]);
35
+ /** Reasons that are positive proof of absence at the leg that reported them. */
36
+ const CONCLUSIVE_ABSENCE = new Set([
37
+ 'not_found',
38
+ 'malformed_payload',
39
+ ]);
40
+ /**
41
+ * The one digest check in the system. Legs that already hold bytes — the byte
42
+ * cache, the self-store, a route resolver, the MCP daemon proxy — call this
43
+ * rather than writing `createHash('sha256')` out again.
44
+ */
45
+ export function verifyArtifactDigest(expectedSha256, bytes) {
46
+ const actualSha256 = createHash('sha256').update(bytes).digest('hex');
47
+ return actualSha256 === expectedSha256 ? { ok: true } : { ok: false, actualSha256 };
48
+ }
49
+ /**
50
+ * Log an IPFS read the retrieval is about to fall through, unless the gateway
51
+ * actually answered "not there" (#3441). A cap refusal is positive evidence the
52
+ * content exists, so it must not read like an ordinary miss.
53
+ */
54
+ function warnIpfsFallThrough(subject, error) {
55
+ // A broken donation is not the ordinary absence #3441 carved out of the
56
+ // warning, so it still warns — but it must not read as a transport failure.
57
+ const classification = error instanceof DonationDecodeError
58
+ ? 'malformed-payload'
59
+ : classifyIpfsFetchFailure(error);
60
+ if (classification === 'not-found')
61
+ return;
62
+ const detail = error instanceof Error ? error.message : String(error);
63
+ console.warn(`[corpus-read] IPFS source for ${subject} could not be used (${classification}), `
64
+ + `falling through to the next source: ${detail}`);
65
+ }
66
+ /**
67
+ * A donation payload that will never decode into this artifact. Distinct from a
68
+ * gateway transport failure: the CID resolved, and what it resolved to is not
69
+ * the requested bytes — including a valid donation for a *different* artifact.
70
+ */
71
+ class DonationDecodeError extends Error {
72
+ name = 'DonationDecodeError';
73
+ }
74
+ /**
75
+ * Decode a `jinn.artifact.donation.v1` payload into its raw bytes.
76
+ *
77
+ * Moved verbatim from `acquire.ts`. The `schemaVersion` check compares against
78
+ * the encoding constant — the published fixtures set both fields to that same
79
+ * string, so the comparison is load-bearing as written and is not "corrected"
80
+ * here.
81
+ */
82
+ function decodeDonationArtifact(raw, expectedSha256) {
83
+ if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
84
+ throw new DonationDecodeError('donation artifact payload is not an object');
85
+ }
86
+ const record = raw;
87
+ if (record['schemaVersion'] !== DONATION_ARTIFACT_ENCODING || record['encoding'] !== DONATION_ARTIFACT_ENCODING) {
88
+ throw new DonationDecodeError('donation artifact payload has unexpected encoding');
89
+ }
90
+ if (record['sha256'] !== expectedSha256) {
91
+ throw new DonationDecodeError('donation artifact sha256 does not match requested artifact');
92
+ }
93
+ if (typeof record['data'] !== 'string') {
94
+ throw new DonationDecodeError('donation artifact payload is missing base64 data');
95
+ }
96
+ return Buffer.from(record['data'], 'base64');
97
+ }
98
+ /** `fetchArtifactContent`'s vocabulary → the primitive's. */
99
+ function originReason(reason) {
100
+ return reason === 'network_error' ? 'unavailable' : reason;
101
+ }
102
+ /** `classifyIpfsFetchFailure`'s hyphenated vocabulary → the primitive's. */
103
+ function ipfsReason(classification) {
104
+ if (classification === 'too-large')
105
+ return 'too_large';
106
+ if (classification === 'not-found')
107
+ return 'not_found';
108
+ return 'unavailable';
109
+ }
110
+ function errorMessage(error) {
111
+ return error instanceof Error ? error.message : String(error);
112
+ }
113
+ /**
114
+ * Fetch an artifact by content address and return its bytes only after they
115
+ * hash to that address.
116
+ *
117
+ * Legs run in the order the daemon chain has always used: the free donated IPFS
118
+ * mirror first, the operator's HTTP origin second. A leg with no locator is
119
+ * *skipped*, not failed. A digest mismatch on any leg fails the whole call and
120
+ * does not fall through — the bytes are wrong or hostile, and the next locator
121
+ * cannot make them right.
122
+ */
123
+ export async function fetchVerifiedArtifact(address, locators, options) {
124
+ const { sha256, artifactType } = address;
125
+ const { sources = [], ipfsGatewayUrl, endpoint, envelopeCid, ownerSafe } = locators;
126
+ const fetchIpfs = options?.deps?.fetchFromIpfs ?? defaultFetchFromIpfs;
127
+ const fetchOrigin = options?.deps?.fetchArtifact ?? fetchArtifactContent;
128
+ const now = options?.deps?.now ?? (() => new Date().toISOString());
129
+ const attempts = [];
130
+ const admit = (source, sourceUri, bytes) => {
131
+ attempts.push({ leg: source, sourceUri, outcome: 'verified' });
132
+ return {
133
+ ok: true,
134
+ artifact: {
135
+ sha256,
136
+ bytes,
137
+ artifactType: artifactType ?? 'unknown',
138
+ provenance: {
139
+ source,
140
+ sourceUri,
141
+ digestAlgorithm: 'sha256',
142
+ fetchedAt: now(),
143
+ ...(ownerSafe ? { sourceOperator: ownerSafe } : {}),
144
+ ...(envelopeCid ? { envelopeCid } : {}),
145
+ attempts,
146
+ },
147
+ },
148
+ };
149
+ };
150
+ const refuse = (leg, sourceUri, actualSha256) => {
151
+ attempts.push({ leg, sourceUri, outcome: 'digest_mismatch' });
152
+ return {
153
+ ok: false,
154
+ sha256,
155
+ reason: 'digest_mismatch',
156
+ retryable: false,
157
+ message: `artifact ${sha256} from ${sourceUri} hashed to ${actualSha256}; `
158
+ + 'the bytes were discarded',
159
+ attempts,
160
+ mismatch: {
161
+ expectedSha256: sha256,
162
+ actualSha256,
163
+ sourceUri,
164
+ ...(ownerSafe ? { sourceOperator: ownerSafe } : {}),
165
+ },
166
+ };
167
+ };
168
+ // 1. Donated IPFS mirror — free and public, so it goes first.
169
+ const ipfsSource = sources.find((source) => source.kind === 'ipfs');
170
+ if (ipfsSource && ipfsGatewayUrl) {
171
+ const sourceUri = `ipfs://${ipfsSource.cid}`;
172
+ try {
173
+ const raw = await fetchIpfs(ipfsGatewayUrl, ipfsSource.cid);
174
+ const bytes = decodeDonationArtifact(raw, sha256);
175
+ const verified = verifyArtifactDigest(sha256, bytes);
176
+ if (!verified.ok)
177
+ return refuse('ipfs', sourceUri, verified.actualSha256);
178
+ return admit('ipfs', sourceUri, bytes);
179
+ }
180
+ catch (err) {
181
+ // Neither a gateway failure nor a malformed donation payload is a verdict
182
+ // on the artifact *globally*, so both fall through and let the next
183
+ // locator answer. They differ in what they proved here: a transport
184
+ // failure learned nothing, while a payload that will never decode into
185
+ // this artifact is absence at this locator — hence the split reason. Both
186
+ // still warn; only a gateway's own "not there" is silent (#3441).
187
+ warnIpfsFallThrough(`donation artifact ${sha256}`, err);
188
+ attempts.push({
189
+ leg: 'ipfs',
190
+ sourceUri,
191
+ outcome: 'failed',
192
+ reason: err instanceof DonationDecodeError
193
+ ? 'malformed_payload'
194
+ : ipfsReason(classifyIpfsFetchFailure(err)),
195
+ message: errorMessage(err),
196
+ });
197
+ }
198
+ }
199
+ else {
200
+ attempts.push({ leg: 'ipfs', sourceUri: '', outcome: 'skipped' });
201
+ }
202
+ // 2. Operator HTTP origin.
203
+ if (endpoint) {
204
+ const sourceUri = buildArtifactUrl(endpoint, sha256);
205
+ let outcome;
206
+ try {
207
+ outcome = await fetchOrigin(endpoint, sha256);
208
+ }
209
+ catch (err) {
210
+ outcome = { ok: false, reason: 'network_error', message: errorMessage(err) };
211
+ }
212
+ if (outcome.ok) {
213
+ const verified = verifyArtifactDigest(sha256, outcome.content);
214
+ if (!verified.ok)
215
+ return refuse('origin', sourceUri, verified.actualSha256);
216
+ return admit('origin', sourceUri, outcome.content);
217
+ }
218
+ attempts.push({
219
+ leg: 'origin',
220
+ sourceUri,
221
+ outcome: 'failed',
222
+ reason: originReason(outcome.reason),
223
+ ...(outcome.message ? { message: outcome.message } : {}),
224
+ });
225
+ }
226
+ else {
227
+ attempts.push({ leg: 'origin', sourceUri: '', outcome: 'skipped' });
228
+ }
229
+ // Nothing verified. Absence is only claimed when every attempted leg said so:
230
+ // one inconclusive leg means we learned nothing about existence (#3441).
231
+ const failures = attempts.filter((attempt) => attempt.outcome === 'failed');
232
+ if (failures.length === 0) {
233
+ // Name why each leg was skipped rather than assuming both locators were
234
+ // absent: donated sources with no gateway to read them through is its own
235
+ // case, and reporting it as "no donated IPFS source" is simply wrong.
236
+ const missing = [
237
+ ipfsSource
238
+ ? 'a donated IPFS source but no gateway URL to read it through'
239
+ : 'no donated IPFS source',
240
+ 'no origin endpoint',
241
+ ];
242
+ return {
243
+ ok: false,
244
+ sha256,
245
+ reason: 'no_locator',
246
+ retryable: false,
247
+ message: `artifact ${sha256} has no locator to try: ${missing.join(', and ')}`,
248
+ attempts,
249
+ };
250
+ }
251
+ // When more than one leg fails inconclusively, the last one wins. Legs run
252
+ // cheapest-first — the opportunistic donated mirror, then the operator's own
253
+ // origin — so the last leg to answer is the artifact's actual home, and its
254
+ // reason is the one the operator can act on.
255
+ // Absence is a set, not one literal: a leg that resolved and served bytes that
256
+ // are permanently not this artifact answered as conclusively as a 404 did.
257
+ const inconclusive = failures.filter((attempt) => !CONCLUSIVE_ABSENCE.has(attempt.reason));
258
+ const reason = (inconclusive.length > 0 ? inconclusive : failures).at(-1).reason;
259
+ return {
260
+ ok: false,
261
+ sha256,
262
+ reason,
263
+ retryable: RETRYABLE_REASONS.has(reason),
264
+ message: `artifact ${sha256} could not be retrieved (${reason}) from `
265
+ + failures.map((attempt) => `${attempt.sourceUri}: ${attempt.reason}`).join('; '),
266
+ attempts,
267
+ };
268
+ }
@@ -1,14 +1,13 @@
1
1
  /**
2
- * Corpus cache helpers.
2
+ * Corpus cache is not a public helper surface.
3
3
  *
4
4
  * The corpus library reuses the existing `network_artifacts` and `served_artifacts`
5
- * tables on the daemon's `Store`. There is no separate corpus cache layer — the
6
- * acquire chain in `acquire.ts` is the only writer/reader. This module exposes
7
- * thin convenience helpers for higher-level code that wants to inspect or
8
- * pre-populate the cache without going through `acquireArtifactContent`.
5
+ * tables on the daemon's `Store`. There is no separate corpus cache API —
6
+ * `acquireArtifactContent` is the only writer/reader, and it digest-checks cached
7
+ * rows before serving them. Do not reintroduce `getCachedArtifact` /
8
+ * `hasCachedArtifact`: they returned unverified `network_artifacts` rows and
9
+ * would reopen the byte-admission gap #4359 closed (#4500).
9
10
  *
10
11
  * Spec: spec/2026-04-30-phase-a-umbrella.md §2.4 (caching semantics).
11
12
  */
12
- import type { CorpusStorePort, NetworkArtifactRow } from './types.js';
13
- export declare function getCachedArtifact(store: CorpusStorePort, sha256: string): NetworkArtifactRow | null;
14
- export declare function hasCachedArtifact(store: CorpusStorePort, sha256: string): boolean;
13
+ export {};
@@ -1,17 +1 @@
1
- /**
2
- * Corpus cache helpers.
3
- *
4
- * The corpus library reuses the existing `network_artifacts` and `served_artifacts`
5
- * tables on the daemon's `Store`. There is no separate corpus cache layer — the
6
- * acquire chain in `acquire.ts` is the only writer/reader. This module exposes
7
- * thin convenience helpers for higher-level code that wants to inspect or
8
- * pre-populate the cache without going through `acquireArtifactContent`.
9
- *
10
- * Spec: spec/2026-04-30-phase-a-umbrella.md §2.4 (caching semantics).
11
- */
12
- export function getCachedArtifact(store, sha256) {
13
- return store.getNetworkArtifact(sha256);
14
- }
15
- export function hasCachedArtifact(store, sha256) {
16
- return store.getNetworkArtifact(sha256) !== null;
17
- }
1
+ export {};
@@ -0,0 +1,9 @@
1
+ /** Environment-variable parsing shared by the corpus read paths. */
2
+ /**
3
+ * `minimum` is 0 where the bound reads `0` as "disabled" (a whole-operation
4
+ * timeout, or a redirect cap of zero meaning "follow none"), and 1 for a byte
5
+ * cap, where zero would not disable anything — it would reject every response
6
+ * as `too_large`. A foot-gun that silently stops all acquisition is worse than
7
+ * ignoring the value, so an out-of-range setting falls back to the default.
8
+ */
9
+ export declare function envInteger(name: string, fallback: number, minimum?: number): number;
@@ -0,0 +1,15 @@
1
+ /** Environment-variable parsing shared by the corpus read paths. */
2
+ /**
3
+ * `minimum` is 0 where the bound reads `0` as "disabled" (a whole-operation
4
+ * timeout, or a redirect cap of zero meaning "follow none"), and 1 for a byte
5
+ * cap, where zero would not disable anything — it would reject every response
6
+ * as `too_large`. A foot-gun that silently stops all acquisition is worse than
7
+ * ignoring the value, so an out-of-range setting falls back to the default.
8
+ */
9
+ export function envInteger(name, fallback, minimum = 0) {
10
+ const raw = process.env[name];
11
+ if (raw === undefined || raw.trim() === '')
12
+ return fallback;
13
+ const parsed = Number(raw);
14
+ return Number.isInteger(parsed) && parsed >= minimum ? parsed : fallback;
15
+ }
@@ -23,6 +23,7 @@
23
23
  * - the body is streamed through a byte counter and abandoned the moment it
24
24
  * exceeds the cap, so an oversized response is never fully buffered.
25
25
  */
26
+ import { envInteger } from './env.js';
26
27
  import { resolvePublicHttpDestination, ProhibitedDestinationError, } from './origin-guard.js';
27
28
  import { pinnedFetch } from './pinned-fetch.js';
28
29
  /** 32 MiB. Artifacts are solution/evidence payloads, not disk images. */
@@ -34,20 +35,6 @@ class TimeoutError extends Error {
34
35
  }
35
36
  class TooLargeError extends Error {
36
37
  }
37
- /**
38
- * `minimum` is 0 where the bound reads `0` as "disabled" (the timeout, and a
39
- * redirect cap of zero meaning "follow none"), and 1 for the byte cap, where
40
- * zero would not disable anything — it would reject every artifact as
41
- * `too_large`. A foot-gun that silently stops all acquisition is worse than
42
- * ignoring the value, so an out-of-range setting falls back to the default.
43
- */
44
- function envInteger(name, fallback, minimum = 0) {
45
- const raw = process.env[name];
46
- if (raw === undefined || raw.trim() === '')
47
- return fallback;
48
- const parsed = Number(raw);
49
- return Number.isInteger(parsed) && parsed >= minimum ? parsed : fallback;
50
- }
51
38
  function envFlag(name) {
52
39
  const raw = process.env[name];
53
40
  if (raw === undefined)
@@ -1,5 +1,5 @@
1
1
  export * from './acquire.js';
2
- export * from './cache.js';
2
+ export * from './artifact-retrieval.js';
3
3
  export * from './capture-meta.js';
4
4
  export * from './create-corpus.js';
5
5
  export * from './fetch-artifact.js';
@@ -1,5 +1,5 @@
1
1
  export * from './acquire.js';
2
- export * from './cache.js';
2
+ export * from './artifact-retrieval.js';
3
3
  export * from './capture-meta.js';
4
4
  export * from './create-corpus.js';
5
5
  export * from './fetch-artifact.js';
@@ -56,6 +56,37 @@ export declare class IpfsFetchFailedError extends Error {
56
56
  * default fetch also tries the `ipfs.io` fallback, which typically stalls into a 504 or an abort
57
57
  * for an unpinned digest rather than answering 404. Such a run classifies `'unavailable'` — the
58
58
  * safe direction, since absence is never claimed without proof of it.
59
+ *
60
+ * That near-unreachability was weighed and is ACCEPTED (#3477). Three loosenings were considered
61
+ * and all three are rejected:
62
+ *
63
+ * - Treat a primary-gateway 404 as authoritative absence. A gateway 404 means *that gateway did
64
+ * not resolve the CID inside its own timeout*, not that the content is absent from the
65
+ * network. Elevating it to a global absence claim is strictly weaker evidence than the current
66
+ * rule, in the exact direction #2647's FAILURE IS NOT ABSENCE exists to forbid.
67
+ * - Surface per-gateway evidence to the caller. Nothing to build: {@link
68
+ * IpfsFetchFailedError.causes} is already a per-candidate array in attempt order, and the
69
+ * aggregate message names each candidate. So the strictness costs no INFORMATION — only the
70
+ * convenience of a single verdict, which any caller needing finer granularity can bypass.
71
+ * - Separate the never-reached fallback out as its own case, distinct from a fallback that
72
+ * answered something other than 404. A candidate the whole-operation deadline never reached
73
+ * is not evidence of absence — the
74
+ * un-attempted gateway could have served it. Counting a non-attempt as absence-supporting is
75
+ * the first loosening with an extra step.
76
+ *
77
+ * The asymmetry is what settles it. Classifying genuine absence as `'unavailable'` costs one
78
+ * warning line and a retry on a later tick: cheap and self-correcting. Classifying a mere
79
+ * non-answer as `'not-found'` costs silent treatment as absent, no retry signal, and a data gap
80
+ * indistinguishable from real data: expensive, permanent, invisible.
81
+ *
82
+ * The strictness is not free: control flow does branch on `'not-found'`, so its cost is noisier
83
+ * `'unavailable'` warnings for content that is genuinely absent, rather than none —
84
+ * `buildFetchIpfsBytes` (returns `undefined` instead of the refusal string) and `narrowIpfsBytes`
85
+ * (silent instead of `logger.warn`) in `operator/src/daemon/composition-root.ts`;
86
+ * `warnIpfsFallThrough` (silent instead of `console.warn`) and `ipfsReason` (reason `not_found`
87
+ * instead of `too_large` / `unavailable`) in `./artifact-retrieval.ts`; and the task, trajectory,
88
+ * and source-bundle reads in `operator/src/conformance/harness.ts` (silent instead of
89
+ * `console.warn`).
59
90
  */
60
91
  export declare function classifyIpfsFetchFailure(error: unknown): 'too-large' | 'not-found' | 'unavailable';
61
92
  export declare function normalizeIpfsGatewayBase(gatewayUrl: string): string;
@@ -74,7 +105,14 @@ export type FetchFromIpfsOptions = {
74
105
  * {@link DEFAULT_MAX_IPFS_RESPONSE_BYTES}. Raise it only for a call site that
75
106
  * legitimately reads larger payloads (#3441) — the default is sized for JSON
76
107
  * envelopes and source files, and a caller that raises it accepts buffering
77
- * that many bytes. Values below 1 fall back to the default.
108
+ * that many bytes. Values below 1 fall back to the env, then the default.
109
+ *
110
+ * An operator may move the *default* with `JINN_IPFS_MAX_RESPONSE_BYTES`
111
+ * (#3453). An explicit bound here wins over it, so the env cannot clamp a
112
+ * call site that already states its own — concretely, it does not reach
113
+ * `fetchTrajectoryFromIpfs`'s 64 MiB (`MAX_TRAJECTORY_IPFS_RESPONSE_BYTES`,
114
+ * `operator/src/adapters/mech/ipfs.ts`). Making it a ceiling instead would
115
+ * make the raise-the-cap intent this option exists for inexpressible.
78
116
  */
79
117
  maxResponseBytes?: number;
80
118
  };
@@ -1,3 +1,4 @@
1
+ import { envInteger } from './env.js';
1
2
  const IPFS_FETCH_TIMEOUT_MS = 15_000;
2
3
  /**
3
4
  * Bound on one whole `fetchFromIpfs` / `fetchBytesFromIpfs` call. The
@@ -79,6 +80,37 @@ export class IpfsFetchFailedError extends Error {
79
80
  * default fetch also tries the `ipfs.io` fallback, which typically stalls into a 504 or an abort
80
81
  * for an unpinned digest rather than answering 404. Such a run classifies `'unavailable'` — the
81
82
  * safe direction, since absence is never claimed without proof of it.
83
+ *
84
+ * That near-unreachability was weighed and is ACCEPTED (#3477). Three loosenings were considered
85
+ * and all three are rejected:
86
+ *
87
+ * - Treat a primary-gateway 404 as authoritative absence. A gateway 404 means *that gateway did
88
+ * not resolve the CID inside its own timeout*, not that the content is absent from the
89
+ * network. Elevating it to a global absence claim is strictly weaker evidence than the current
90
+ * rule, in the exact direction #2647's FAILURE IS NOT ABSENCE exists to forbid.
91
+ * - Surface per-gateway evidence to the caller. Nothing to build: {@link
92
+ * IpfsFetchFailedError.causes} is already a per-candidate array in attempt order, and the
93
+ * aggregate message names each candidate. So the strictness costs no INFORMATION — only the
94
+ * convenience of a single verdict, which any caller needing finer granularity can bypass.
95
+ * - Separate the never-reached fallback out as its own case, distinct from a fallback that
96
+ * answered something other than 404. A candidate the whole-operation deadline never reached
97
+ * is not evidence of absence — the
98
+ * un-attempted gateway could have served it. Counting a non-attempt as absence-supporting is
99
+ * the first loosening with an extra step.
100
+ *
101
+ * The asymmetry is what settles it. Classifying genuine absence as `'unavailable'` costs one
102
+ * warning line and a retry on a later tick: cheap and self-correcting. Classifying a mere
103
+ * non-answer as `'not-found'` costs silent treatment as absent, no retry signal, and a data gap
104
+ * indistinguishable from real data: expensive, permanent, invisible.
105
+ *
106
+ * The strictness is not free: control flow does branch on `'not-found'`, so its cost is noisier
107
+ * `'unavailable'` warnings for content that is genuinely absent, rather than none —
108
+ * `buildFetchIpfsBytes` (returns `undefined` instead of the refusal string) and `narrowIpfsBytes`
109
+ * (silent instead of `logger.warn`) in `operator/src/daemon/composition-root.ts`;
110
+ * `warnIpfsFallThrough` (silent instead of `console.warn`) and `ipfsReason` (reason `not_found`
111
+ * instead of `too_large` / `unavailable`) in `./artifact-retrieval.ts`; and the task, trajectory,
112
+ * and source-bundle reads in `operator/src/conformance/harness.ts` (silent instead of
113
+ * `console.warn`).
82
114
  */
83
115
  export function classifyIpfsFetchFailure(error) {
84
116
  const causes = error instanceof IpfsFetchFailedError ? error.causes : [error];
@@ -93,20 +125,32 @@ export function normalizeIpfsGatewayBase(gatewayUrl) {
93
125
  let normalized = gatewayUrl.trim();
94
126
  if (normalized === '')
95
127
  normalized = 'https://gateway.autonolas.tech';
96
- // Drop any userinfo at the source. `fetch` rejects a credentialed URL
97
- // outright, and its own error message quotes the URL back — so a gateway
98
- // configured Infura-style would otherwise put its secret into every
99
- // aggregated fetch error, which callers log.
100
128
  try {
101
129
  const parsed = new URL(normalized);
102
- if (parsed.username !== '' || parsed.password !== '') {
103
- parsed.username = '';
104
- parsed.password = '';
105
- normalized = parsed.toString();
106
- }
130
+ // Drop any userinfo at the source. `fetch` rejects a credentialed URL
131
+ // outright, and its own error message quotes the URL back — so a gateway
132
+ // configured Infura-style would otherwise put its secret into every
133
+ // aggregated fetch error, which callers log.
134
+ parsed.username = '';
135
+ parsed.password = '';
136
+ // A fragment is never transmitted to a server, and treating it as part of
137
+ // the base corrupted the `/ipfs` suffix test into appending a second
138
+ // segment (`…/ipfs#frag` → `…/ipfs#frag/ipfs/`). Dropping it is the only
139
+ // reading that is both correct and lossless in transit.
140
+ parsed.hash = '';
141
+ // Normalize on the parsed path, not the raw string: a query-bearing base
142
+ // otherwise had `/ipfs/` spliced on after the query, putting the CID
143
+ // inside the query string (#3452).
144
+ let path = parsed.pathname.replace(/\/+$/, '');
145
+ if (!path.toLowerCase().endsWith('/ipfs'))
146
+ path = `${path}/ipfs`;
147
+ parsed.pathname = `${path}/`;
148
+ return parsed.toString();
107
149
  }
108
150
  catch {
109
- // Not an absolute URL; leave it to the caller's own failure path.
151
+ // Not an absolute URL; leave it to the caller's own failure path, which
152
+ // reports it as a per-candidate `candidate URL could not be parsed`
153
+ // instead of escaping as a bare TypeError.
110
154
  }
111
155
  normalized = normalized.replace(/\/+$/, '');
112
156
  if (!normalized.toLowerCase().endsWith('/ipfs'))
@@ -190,7 +234,51 @@ async function discardBody(response) {
190
234
  // A body that refuses to cancel is not worth failing the request over.
191
235
  }
192
236
  }
193
- /** Read a response body as bytes, refusing anything past `maxBytes`. */
237
+ /**
238
+ * The exact decoded length this response promises, or `undefined` when it
239
+ * promises nothing usable (#3759).
240
+ *
241
+ * `content-encoding` is the trap: with one set, `content-length` describes
242
+ * *wire* bytes, and the reader hands back the decoded ones — a gzipped 200 KB
243
+ * JSON body declares ~230. Pre-sizing from that is not a pessimization, it is
244
+ * a buffer guaranteed to overflow, so the fast path is gated on the encoding
245
+ * being absent or `identity`. The raw header is read rather than
246
+ * `Number(header)` because `Number(null)` is `0`, which cannot be told from a
247
+ * genuinely declared zero.
248
+ */
249
+ function exactDecodedLength(response, maxBytes) {
250
+ const encoding = response.headers.get('content-encoding');
251
+ if (encoding !== null && encoding.trim().toLowerCase() !== 'identity')
252
+ return undefined;
253
+ const raw = response.headers.get('content-length');
254
+ if (raw === null || raw.trim() === '')
255
+ return undefined;
256
+ const declared = Number(raw);
257
+ if (!Number.isInteger(declared) || declared < 0 || declared > maxBytes)
258
+ return undefined;
259
+ return declared;
260
+ }
261
+ /**
262
+ * Read a response body as bytes, refusing anything past `maxBytes`.
263
+ *
264
+ * Peak resident bytes, not memcpy count, is what the pre-sizing below buys.
265
+ * The stream has already allocated each chunk either way, and the join is the
266
+ * single copy in both designs — but the chunk-list path holds every chunk live
267
+ * *while* it allocates and fills the joined buffer, so it peaks at roughly
268
+ * twice the payload: ~128 MiB at `fetchTrajectoryFromIpfs`'s 64 MiB bound
269
+ * (measured), against ~1x live for the pre-sized path.
270
+ *
271
+ * The honest caveat: the fast path is gated on the absence of a content
272
+ * encoding, and gateways generally compress JSON — so it will rarely engage on
273
+ * the very trajectory read that motivated the bound. The win is real for the
274
+ * raw-bytes path (source files, sealed documents, envelopes) and
275
+ * correct-but-inert for compressed JSON.
276
+ *
277
+ * Out of scope, and larger than what this removes: `fetchJson` decodes these
278
+ * bytes into a UTF-16 string (up to 2x the byte length) and then `JSON.parse`s
279
+ * that into an object graph, all while the bytes are still live. Bringing that
280
+ * down means streaming JSON parsing, which is a rewrite rather than a fix.
281
+ */
194
282
  async function readBoundedBytes(response, maxBytes) {
195
283
  const declared = Number(response.headers.get('content-length'));
196
284
  if (Number.isFinite(declared) && declared > maxBytes) {
@@ -201,6 +289,9 @@ async function readBoundedBytes(response, maxBytes) {
201
289
  if (!body)
202
290
  return new Uint8Array(0);
203
291
  const reader = body.getReader();
292
+ const exact = exactDecodedLength(response, maxBytes);
293
+ let buffer = exact === undefined ? undefined : new Uint8Array(exact);
294
+ let written = 0;
204
295
  const chunks = [];
205
296
  let total = 0;
206
297
  try {
@@ -211,9 +302,27 @@ async function readBoundedBytes(response, maxBytes) {
211
302
  if (!value)
212
303
  continue;
213
304
  total += value.byteLength;
305
+ // The cumulative counter stays the authority on the cap. The pre-sized
306
+ // buffer's own length is never it — a lying `content-length` must not be
307
+ // able to widen or narrow the bound.
214
308
  if (total > maxBytes) {
215
309
  throw new IpfsResponseTooLargeError(maxBytes);
216
310
  }
311
+ if (buffer !== undefined) {
312
+ if (written + value.byteLength <= buffer.length) {
313
+ buffer.set(value, written);
314
+ written += value.byteLength;
315
+ continue;
316
+ }
317
+ // The body outran what it declared, on an identity response. Fall back
318
+ // in place: what was written becomes the first chunk-list entry and
319
+ // the read continues exactly as it did before. Peak on this anomalous
320
+ // path is 2x — today's behaviour, so no regression and no new failure
321
+ // mode. (`subarray` is right here: the pre-sized buffer is dropped
322
+ // straight after, so retaining its backing store costs nothing.)
323
+ chunks.push(buffer.subarray(0, written));
324
+ buffer = undefined;
325
+ }
217
326
  chunks.push(value);
218
327
  }
219
328
  }
@@ -225,6 +334,15 @@ async function readBoundedBytes(response, maxBytes) {
225
334
  // Already terminal; the read result (or throw) above is what matters.
226
335
  }
227
336
  }
337
+ if (buffer !== undefined) {
338
+ // A short body is not reachable through `fetch` — undici rejects a body
339
+ // that ends early with `TypeError: terminated` — but returning trailing
340
+ // zeros would be wrong if it ever were, and the guard costs one branch.
341
+ // `slice`, never `subarray`: a view would retain the whole pre-sized
342
+ // backing store and silently defeat this change on the one path where it
343
+ // matters.
344
+ return written === buffer.length ? buffer : buffer.slice(0, written);
345
+ }
228
346
  const joined = new Uint8Array(total);
229
347
  let offset = 0;
230
348
  for (const chunk of chunks) {
@@ -303,10 +421,14 @@ async function fetchBytes(url, signal, maxBytes) {
303
421
  }
304
422
  function resolveMaxResponseBytes(opts) {
305
423
  const requested = opts?.maxResponseBytes;
306
- if (typeof requested !== 'number' || !Number.isFinite(requested) || requested < 1) {
307
- return DEFAULT_MAX_IPFS_RESPONSE_BYTES;
308
- }
309
- return Math.floor(requested);
424
+ if (typeof requested === 'number' && Number.isFinite(requested) && requested >= 1) {
425
+ return Math.floor(requested);
426
+ }
427
+ // The env replaces the default as the fallback; it is not a second
428
+ // validation layer, so an out-of-range *option* still falls through to it.
429
+ // `minimum = 1` is what keeps the cap from being disabled into an unbounded
430
+ // read: `0` and every other out-of-range value land on the default.
431
+ return envInteger('JINN_IPFS_MAX_RESPONSE_BYTES', DEFAULT_MAX_IPFS_RESPONSE_BYTES, 1);
310
432
  }
311
433
  function resolveFallbackGatewayBases(opts) {
312
434
  if (opts?.fallbackGatewayBase === false)
@@ -347,6 +469,24 @@ function resolveGatewayCandidateUrl(baseUrl, cidPath) {
347
469
  if (resolved.origin !== base.origin || !resolved.pathname.startsWith(base.pathname)) {
348
470
  return { reason: 'candidate URL escapes the gateway path prefix' };
349
471
  }
472
+ // `new URL(cidPath, base)` drops the base's query unconditionally, which
473
+ // strips the API key off every request to an authenticated gateway (#3452).
474
+ // Re-attach it *after* the guard, so a manifest-supplied CID path can never
475
+ // influence the check, and only when the base has one, so every query-free
476
+ // base issues a byte-identical request. The base wins over any query the CID
477
+ // path carries: two query strings cannot be merged unambiguously, and
478
+ // letting manifest text override an operator's gateway credentials is the
479
+ // wrong direction.
480
+ //
481
+ // Two known limits, both narrow and neither fixed here. A *relative*
482
+ // redirect `Location` is resolved with `new URL(location, current)`, which
483
+ // drops the query, so the key is not forwarded across such a hop — gateways
484
+ // that redirect path form to subdomain form send an absolute `Location`, and
485
+ // `assertRedirectAllowed` already pins the host family and port. And
486
+ // `displayUrl` stays `origin + pathname`, which is what keeps the credential
487
+ // out of logs now that the request actually carries it.
488
+ if (base.search !== '')
489
+ resolved.search = base.search;
350
490
  return { url: resolved };
351
491
  }
352
492
  /**
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@jinn-network/core",
3
- "version": "0.1.2-canary.bebeb033",
3
+ "version": "0.1.2-canary.c8da24a8",
4
4
  "description": "Domain core for Jinn — evidence/contribution stores, scrub, trajectory parsing, and corpus reads. Independent of operator/src.",
5
5
  "type": "module",
6
6
  "packageManager": "yarn@4.13.0",
@@ -43,7 +43,7 @@
43
43
  },
44
44
  "dependencies": {
45
45
  "@huggingface/transformers": "^4.2.0",
46
- "@jinn-network/plugin": "0.1.2-canary.bebeb033",
46
+ "@jinn-network/plugin": "0.1.2-canary.c8da24a8",
47
47
  "@lmoe/gliner-onnx": "0.1.0",
48
48
  "@noble/hashes": "^2.2.0",
49
49
  "@secretlint/core": "^13.0.2",