@jinn-network/core 0.1.2-canary.2b991f91 → 0.1.2-canary.402cd81b
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/captured-task.d.ts +1 -1
- package/dist/captured-task.js +1 -1
- package/dist/corpus-read/acquire.d.ts +6 -0
- package/dist/corpus-read/acquire.js +49 -70
- package/dist/corpus-read/artifact-retrieval.d.ts +156 -0
- package/dist/corpus-read/artifact-retrieval.js +246 -0
- package/dist/corpus-read/fetch-artifact.d.ts +47 -2
- package/dist/corpus-read/fetch-artifact.js +190 -9
- package/dist/corpus-read/index.d.ts +3 -0
- package/dist/corpus-read/index.js +3 -0
- package/dist/corpus-read/ipfs.d.ts +74 -0
- package/dist/corpus-read/ipfs.js +372 -24
- package/dist/corpus-read/origin-guard.d.ts +75 -0
- package/dist/corpus-read/origin-guard.js +265 -0
- package/dist/corpus-read/pinned-fetch.d.ts +42 -0
- package/dist/corpus-read/pinned-fetch.js +121 -0
- package/dist/envelope.d.ts +2 -2
- package/dist/envelope.js +2 -2
- package/dist/skill-artifact.d.ts +2 -2
- package/dist/skill-artifact.js +2 -2
- package/package.json +4 -4
package/dist/captured-task.d.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Captured-task schema — the raw, pre-scrub input to the harness capture path.
|
|
3
3
|
*
|
|
4
4
|
* Extracted from the former client harness layer in C2 (#1833):
|
|
5
|
-
* the parse half of capture.ts has no scrub / `
|
|
5
|
+
* the parse half of capture.ts has no scrub / `operator/src` dependency (only
|
|
6
6
|
* zod + the two envelope enums), so it moved into `@jinn-network/core`.
|
|
7
7
|
* C5 subsequently moved the scrub implementation alongside it while leaving
|
|
8
8
|
* capture orchestration in the layer.
|
package/dist/captured-task.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Captured-task schema — the raw, pre-scrub input to the harness capture path.
|
|
3
3
|
*
|
|
4
4
|
* Extracted from the former client harness layer in C2 (#1833):
|
|
5
|
-
* the parse half of capture.ts has no scrub / `
|
|
5
|
+
* the parse half of capture.ts has no scrub / `operator/src` dependency (only
|
|
6
6
|
* zod + the two envelope enums), so it moved into `@jinn-network/core`.
|
|
7
7
|
* C5 subsequently moved the scrub implementation alongside it while leaving
|
|
8
8
|
* capture orchestration in the layer.
|
|
@@ -3,6 +3,12 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Always hash-verifies before persisting to cache.
|
|
5
5
|
*
|
|
6
|
+
* The two *network* legs — the donated IPFS mirror and the operator origin —
|
|
7
|
+
* are not implemented here: they are `fetchVerifiedArtifact`, the keyless
|
|
8
|
+
* retrieval primitive (#4179). What remains in this file is the store-aware
|
|
9
|
+
* composition around it: the cache read, the self-store mirror, the route
|
|
10
|
+
* resolver, and the cache writes whose ordering the daemon owns.
|
|
11
|
+
*
|
|
6
12
|
* Spec §2.3 step 4-6.
|
|
7
13
|
*/
|
|
8
14
|
import { type AcquireResult } from './fetch-artifact.js';
|
|
@@ -3,34 +3,34 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Always hash-verifies before persisting to cache.
|
|
5
5
|
*
|
|
6
|
+
* The two *network* legs — the donated IPFS mirror and the operator origin —
|
|
7
|
+
* are not implemented here: they are `fetchVerifiedArtifact`, the keyless
|
|
8
|
+
* retrieval primitive (#4179). What remains in this file is the store-aware
|
|
9
|
+
* composition around it: the cache read, the self-store mirror, the route
|
|
10
|
+
* resolver, and the cache writes whose ordering the daemon owns.
|
|
11
|
+
*
|
|
6
12
|
* Spec §2.3 step 4-6.
|
|
7
13
|
*/
|
|
8
|
-
import { createHash } from 'node:crypto';
|
|
9
14
|
import { fetchArtifactContent } from './fetch-artifact.js';
|
|
15
|
+
import { fetchVerifiedArtifact, verifyArtifactDigest } from './artifact-retrieval.js';
|
|
10
16
|
import { AcquireError, HashMismatchError } from './types.js';
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
function
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
throw new Error('donation artifact sha256 does not match requested artifact');
|
|
26
|
-
}
|
|
27
|
-
if (typeof record['data'] !== 'string') {
|
|
28
|
-
throw new Error('donation artifact payload is missing base64 data');
|
|
29
|
-
}
|
|
30
|
-
return Buffer.from(record['data'], 'base64');
|
|
17
|
+
/**
|
|
18
|
+
* Adapt a legacy `AcquireFn` to the primitive's clean seam. The legacy shape —
|
|
19
|
+
* a third `privateKey` parameter and a `Buffer | null` return — exists for
|
|
20
|
+
* older test fakes and stays confined to this file.
|
|
21
|
+
*/
|
|
22
|
+
function toAcquireResultFn(acquireFn, privateKey) {
|
|
23
|
+
return async (endpoint, sha256) => {
|
|
24
|
+
const raw = await acquireFn(endpoint, sha256, privateKey);
|
|
25
|
+
if (raw === null)
|
|
26
|
+
return { ok: false, reason: 'not_found', message: 'origin returned null (404 / not found)' };
|
|
27
|
+
if (Buffer.isBuffer(raw))
|
|
28
|
+
return { ok: true, content: raw };
|
|
29
|
+
return raw;
|
|
30
|
+
};
|
|
31
31
|
}
|
|
32
32
|
export async function acquireArtifactContent(args) {
|
|
33
|
-
const { sha256, artifactType, access, store, selfSafeAddress, privateKey, routeResolver, envelopeCid, sources = [], ipfsGatewayUrl, ownerSafe, acquireFn = (endpoint, sha256) => fetchArtifactContent(endpoint, sha256), fetchFromIpfs
|
|
33
|
+
const { sha256, artifactType, access, store, selfSafeAddress, privateKey, routeResolver, envelopeCid, sources = [], ipfsGatewayUrl, ownerSafe, acquireFn = (endpoint, sha256) => fetchArtifactContent(endpoint, sha256), fetchFromIpfs, } = args;
|
|
34
34
|
const now = () => new Date().toISOString();
|
|
35
35
|
// 1. Cache hit
|
|
36
36
|
const cached = store.getNetworkArtifact(sha256);
|
|
@@ -46,16 +46,14 @@ export async function acquireArtifactContent(args) {
|
|
|
46
46
|
sourceOperator: cached.sourceOperator ?? undefined,
|
|
47
47
|
};
|
|
48
48
|
}
|
|
49
|
-
// 2. Public IPFS donation source.
|
|
49
|
+
// 2. Public IPFS donation source. The primitive owns the fetch, the decode,
|
|
50
|
+
// the verification, and the #3441 fall-through warning; the cache write and
|
|
51
|
+
// the mismatch-is-fatal decision stay here, where the store lives.
|
|
50
52
|
const ipfsSource = sources.find((source) => source.kind === 'ipfs');
|
|
51
53
|
if (ipfsSource && ipfsGatewayUrl) {
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
const bytes =
|
|
55
|
-
const actualSha = sha256Hex(bytes);
|
|
56
|
-
if (actualSha !== sha256) {
|
|
57
|
-
throw new HashMismatchError(sha256, actualSha, 'ipfs', ownerSafe);
|
|
58
|
-
}
|
|
54
|
+
const retrieved = await fetchVerifiedArtifact({ sha256, artifactType }, { sources, ipfsGatewayUrl, ownerSafe, envelopeCid }, { deps: { fetchFromIpfs } });
|
|
55
|
+
if (retrieved.ok) {
|
|
56
|
+
const bytes = retrieved.artifact.bytes;
|
|
59
57
|
const ts = now();
|
|
60
58
|
store.saveNetworkArtifact({
|
|
61
59
|
sha256,
|
|
@@ -78,13 +76,13 @@ export async function acquireArtifactContent(args) {
|
|
|
78
76
|
sourceOperator: ownerSafe,
|
|
79
77
|
};
|
|
80
78
|
}
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
throw err;
|
|
84
|
-
// Donated IPFS is an opportunistic fast path. Gateway failures and
|
|
85
|
-
// malformed donation payloads should not block the acquisition chain;
|
|
86
|
-
// only verified byte hash mismatches remain fatal.
|
|
79
|
+
if (retrieved.reason === 'digest_mismatch') {
|
|
80
|
+
throw new HashMismatchError(sha256, retrieved.mismatch.actualSha256, 'ipfs', ownerSafe);
|
|
87
81
|
}
|
|
82
|
+
// Donated IPFS is an opportunistic fast path: gateway failures and
|
|
83
|
+
// malformed donation payloads fall through to the next source. Anything
|
|
84
|
+
// that was not proven absence has already reached the operator as a
|
|
85
|
+
// warning from inside the primitive (#3441).
|
|
88
86
|
}
|
|
89
87
|
// 3. Self-store fast path
|
|
90
88
|
if (ownerSafe && ownerSafe.toLowerCase() === selfSafeAddress.toLowerCase()) {
|
|
@@ -95,9 +93,9 @@ export async function acquireArtifactContent(args) {
|
|
|
95
93
|
// propagate the bad bytes into network_artifacts where peers can
|
|
96
94
|
// fetch them. Throwing closes the cache-poisoning gap; the self-store
|
|
97
95
|
// path is now hash-equivalent to origin / route-resolver.
|
|
98
|
-
const
|
|
99
|
-
if (
|
|
100
|
-
throw new HashMismatchError(sha256,
|
|
96
|
+
const verified = verifyArtifactDigest(sha256, own.content);
|
|
97
|
+
if (!verified.ok) {
|
|
98
|
+
throw new HashMismatchError(sha256, verified.actualSha256, 'self-store', selfSafeAddress);
|
|
101
99
|
}
|
|
102
100
|
// Mirror into cache so peer asks for the same content can hit cache (provenance: self-store-mirror).
|
|
103
101
|
const ts = now();
|
|
@@ -125,9 +123,9 @@ export async function acquireArtifactContent(args) {
|
|
|
125
123
|
try {
|
|
126
124
|
const out = await routeResolver.resolve({ sha256, access, requesterSafe: selfSafeAddress });
|
|
127
125
|
if (out) {
|
|
128
|
-
const
|
|
129
|
-
if (
|
|
130
|
-
throw new HashMismatchError(sha256,
|
|
126
|
+
const verified = verifyArtifactDigest(sha256, out.bytes);
|
|
127
|
+
if (!verified.ok) {
|
|
128
|
+
throw new HashMismatchError(sha256, verified.actualSha256, 'route-resolver', out.sourceOperator);
|
|
131
129
|
}
|
|
132
130
|
const ts = now();
|
|
133
131
|
store.saveNetworkArtifact({
|
|
@@ -158,36 +156,17 @@ export async function acquireArtifactContent(args) {
|
|
|
158
156
|
}
|
|
159
157
|
}
|
|
160
158
|
// 5. Origin fetch
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
// union from fetchArtifactContent. Buffer.isBuffer guards before we
|
|
166
|
-
// duck-type into the union.
|
|
167
|
-
if (raw === null || Buffer.isBuffer(raw)) {
|
|
168
|
-
bytes = raw;
|
|
169
|
-
}
|
|
170
|
-
else if (raw.ok === true) {
|
|
171
|
-
bytes = raw.content;
|
|
159
|
+
const retrieved = await fetchVerifiedArtifact({ sha256, artifactType }, { endpoint: access.endpoint, ownerSafe, envelopeCid }, { deps: { fetchArtifact: toAcquireResultFn(acquireFn, privateKey) } });
|
|
160
|
+
if (!retrieved.ok) {
|
|
161
|
+
if (retrieved.reason === 'digest_mismatch') {
|
|
162
|
+
throw new HashMismatchError(sha256, retrieved.mismatch.actualSha256, 'origin', ownerSafe);
|
|
172
163
|
}
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
}
|
|
178
|
-
}
|
|
179
|
-
catch (err) {
|
|
180
|
-
if (err instanceof AcquireError)
|
|
181
|
-
throw err;
|
|
182
|
-
throw new AcquireError(sha256, 'origin fetch failed', err);
|
|
183
|
-
}
|
|
184
|
-
if (!bytes) {
|
|
185
|
-
throw new AcquireError(sha256, 'origin returned null (404 / not found)');
|
|
186
|
-
}
|
|
187
|
-
const actualSha = sha256Hex(bytes);
|
|
188
|
-
if (actualSha !== sha256) {
|
|
189
|
-
throw new HashMismatchError(sha256, actualSha, 'origin', ownerSafe);
|
|
164
|
+
// Preserve the reason token in the message so callers and the daemon route
|
|
165
|
+
// can surface which failure this was.
|
|
166
|
+
const attempt = retrieved.attempts.find((entry) => entry.leg === 'origin');
|
|
167
|
+
throw new AcquireError(sha256, `origin fetch failed: ${retrieved.reason}${attempt?.message ? ` (${attempt.message})` : ''}`, retrieved);
|
|
190
168
|
}
|
|
169
|
+
const bytes = retrieved.artifact.bytes;
|
|
191
170
|
const ts = now();
|
|
192
171
|
store.saveNetworkArtifact({
|
|
193
172
|
sha256,
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Artifact retrieval primitive — fetch by content address, verify, return bytes.
|
|
3
|
+
*
|
|
4
|
+
* One function fetches an artifact from the network legs an envelope names and
|
|
5
|
+
* hands back its bytes only after they hash to the recorded sha256, together
|
|
6
|
+
* with the provenance of the leg that produced them. A digest mismatch fails
|
|
7
|
+
* closed: the failure arm of the result union has no bytes on it at all.
|
|
8
|
+
*
|
|
9
|
+
* Keyless and filesystem-neutral by construction. The primitive itself neither
|
|
10
|
+
* accepts nor touches key material or the filesystem: it takes no store, no
|
|
11
|
+
* signer, no private key, and no path, and it performs no write of any kind.
|
|
12
|
+
* The one channel through which such material can reach a call is the caller's
|
|
13
|
+
* own `deps` seam closure — `acquire.ts` legitimately supplies one that holds a
|
|
14
|
+
* private key — and that closure is opaque here: it is invoked, never read,
|
|
15
|
+
* never logged, and never surfaced in provenance.
|
|
16
|
+
* `test/architecture/keyless-primitive.test.ts` holds the module graph to the
|
|
17
|
+
* same promise.
|
|
18
|
+
*
|
|
19
|
+
* Where the returned bytes land is the caller's decision. The daemon chain
|
|
20
|
+
* (`acquire.ts`) composes this with its byte cache; the CLI hands them to the
|
|
21
|
+
* operator; the MCP handler proxies to a daemon that runs it. None of that is
|
|
22
|
+
* this function's business.
|
|
23
|
+
*
|
|
24
|
+
* Issue #4179 (umbrella #2448 / journey spec B0c).
|
|
25
|
+
*/
|
|
26
|
+
import { type AcquireResult } from './fetch-artifact.js';
|
|
27
|
+
import type { ArtifactSource } from './types.js';
|
|
28
|
+
/**
|
|
29
|
+
* Content address of a corpus artifact: the sha256 of the bytes themselves, as
|
|
30
|
+
* recorded in the envelope's `artifact.sha256` — never an IPFS CID (artifacts
|
|
31
|
+
* stopped having CIDs post jinn-mono-vy37.1.2).
|
|
32
|
+
*/
|
|
33
|
+
export interface ArtifactAddress {
|
|
34
|
+
/** 64-char lowercase hex. The one thing that is trusted. */
|
|
35
|
+
sha256: string;
|
|
36
|
+
/**
|
|
37
|
+
* Declared type from the envelope. Carried into the result for the caller's
|
|
38
|
+
* convenience and never consulted during verification — an attacker-supplied
|
|
39
|
+
* label cannot influence whether bytes are admitted.
|
|
40
|
+
*/
|
|
41
|
+
artifactType?: string;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Where the bytes might be. Every field is manifest-supplied and therefore
|
|
45
|
+
* attacker-controlled; the destination policy and the byte/redirect/deadline
|
|
46
|
+
* caps in `fetch-artifact.ts` and `ipfs.ts` are what make that safe.
|
|
47
|
+
*/
|
|
48
|
+
export interface ArtifactLocators {
|
|
49
|
+
/** Donated IPFS sources — the envelope's `artifact.sources` entries. */
|
|
50
|
+
sources?: readonly ArtifactSource[];
|
|
51
|
+
/** Gateway base for the IPFS leg. The leg is skipped when this is absent. */
|
|
52
|
+
ipfsGatewayUrl?: string;
|
|
53
|
+
/** HTTP origin — the envelope's `artifact.access.endpoint`. */
|
|
54
|
+
endpoint?: string;
|
|
55
|
+
/** Envelope CID this artifact was named by. Provenance only. */
|
|
56
|
+
envelopeCid?: string;
|
|
57
|
+
/** Publishing operator's Safe, when known. Provenance and attribution only. */
|
|
58
|
+
ownerSafe?: string;
|
|
59
|
+
}
|
|
60
|
+
export interface FetchVerifiedArtifactOptions {
|
|
61
|
+
/**
|
|
62
|
+
* Injection seams, mirroring the `acquire.ts` precedent so tests need no
|
|
63
|
+
* module mock. A caller that needs to bound or redirect a leg replaces the
|
|
64
|
+
* leg — there is no separate options passthrough, because no call site wants
|
|
65
|
+
* the default transport with different bounds.
|
|
66
|
+
*/
|
|
67
|
+
deps?: {
|
|
68
|
+
fetchArtifact?: (endpoint: string, sha256: string) => Promise<AcquireResult>;
|
|
69
|
+
fetchFromIpfs?: (gatewayUrl: string, cid: string) => Promise<unknown>;
|
|
70
|
+
/** Clock seam for `fetchedAt`. */
|
|
71
|
+
now?: () => string;
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
/** One leg's outcome, recorded whether it succeeded, was skipped, or failed. */
|
|
75
|
+
export interface ArtifactFetchAttempt {
|
|
76
|
+
readonly leg: 'ipfs' | 'origin';
|
|
77
|
+
/** `ipfs://<cid>` or the exact URL `buildArtifactUrl` produced; empty when skipped. */
|
|
78
|
+
readonly sourceUri: string;
|
|
79
|
+
readonly outcome: 'verified' | 'skipped' | 'failed' | 'digest_mismatch';
|
|
80
|
+
/** Present on `failed`; the classified reason, never a raw string match. */
|
|
81
|
+
readonly reason?: ArtifactFetchFailureReason;
|
|
82
|
+
readonly message?: string;
|
|
83
|
+
}
|
|
84
|
+
export interface ArtifactProvenance {
|
|
85
|
+
/** Which leg produced the admitted bytes. */
|
|
86
|
+
readonly source: 'ipfs' | 'origin';
|
|
87
|
+
readonly sourceUri: string;
|
|
88
|
+
/** Always 'sha256'. Explicit so the record does not silently mean "some digest". */
|
|
89
|
+
readonly digestAlgorithm: 'sha256';
|
|
90
|
+
/** ISO-8601 instant the bytes were verified, not when they were requested. */
|
|
91
|
+
readonly fetchedAt: string;
|
|
92
|
+
readonly sourceOperator?: string;
|
|
93
|
+
readonly envelopeCid?: string;
|
|
94
|
+
/** Every leg tried, in order. Legible even on the happy path. */
|
|
95
|
+
readonly attempts: readonly ArtifactFetchAttempt[];
|
|
96
|
+
}
|
|
97
|
+
export interface VerifiedArtifact {
|
|
98
|
+
readonly sha256: string;
|
|
99
|
+
/** Verified bytes, in memory. Never a path, never a handle. */
|
|
100
|
+
readonly bytes: Buffer;
|
|
101
|
+
readonly artifactType: string;
|
|
102
|
+
readonly provenance: ArtifactProvenance;
|
|
103
|
+
}
|
|
104
|
+
export type ArtifactFetchFailureReason =
|
|
105
|
+
/** Bytes arrived and did not hash to the address. */
|
|
106
|
+
'digest_mismatch'
|
|
107
|
+
/** Every attempted leg answered, and every answer was absence. */
|
|
108
|
+
| 'not_found'
|
|
109
|
+
/** Nothing was supplied to try. */
|
|
110
|
+
| 'no_locator'
|
|
111
|
+
/** Destination policy refused an origin or a redirect hop. */
|
|
112
|
+
| 'blocked' | 'too_large' | 'timeout'
|
|
113
|
+
/** Transport failure — nothing was learned about whether the content exists. */
|
|
114
|
+
| 'unavailable';
|
|
115
|
+
export interface FetchVerifiedArtifactFailure {
|
|
116
|
+
readonly ok: false;
|
|
117
|
+
readonly sha256: string;
|
|
118
|
+
readonly reason: ArtifactFetchFailureReason;
|
|
119
|
+
/** Derived from `reason`, never caller-supplied. */
|
|
120
|
+
readonly retryable: boolean;
|
|
121
|
+
readonly message: string;
|
|
122
|
+
readonly attempts: readonly ArtifactFetchAttempt[];
|
|
123
|
+
/** Present only on `digest_mismatch`. Evidence, not content. */
|
|
124
|
+
readonly mismatch?: {
|
|
125
|
+
readonly expectedSha256: string;
|
|
126
|
+
readonly actualSha256: string;
|
|
127
|
+
readonly sourceUri: string;
|
|
128
|
+
readonly sourceOperator?: string;
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
export type FetchVerifiedArtifactResult = {
|
|
132
|
+
readonly ok: true;
|
|
133
|
+
readonly artifact: VerifiedArtifact;
|
|
134
|
+
} | FetchVerifiedArtifactFailure;
|
|
135
|
+
/**
|
|
136
|
+
* The one digest check in the system. Legs that already hold bytes — the byte
|
|
137
|
+
* cache, the self-store, a route resolver, the MCP daemon proxy — call this
|
|
138
|
+
* rather than writing `createHash('sha256')` out again.
|
|
139
|
+
*/
|
|
140
|
+
export declare function verifyArtifactDigest(expectedSha256: string, bytes: Buffer): {
|
|
141
|
+
ok: true;
|
|
142
|
+
} | {
|
|
143
|
+
ok: false;
|
|
144
|
+
actualSha256: string;
|
|
145
|
+
};
|
|
146
|
+
/**
|
|
147
|
+
* Fetch an artifact by content address and return its bytes only after they
|
|
148
|
+
* hash to that address.
|
|
149
|
+
*
|
|
150
|
+
* Legs run in the order the daemon chain has always used: the free donated IPFS
|
|
151
|
+
* mirror first, the operator's HTTP origin second. A leg with no locator is
|
|
152
|
+
* *skipped*, not failed. A digest mismatch on any leg fails the whole call and
|
|
153
|
+
* does not fall through — the bytes are wrong or hostile, and the next locator
|
|
154
|
+
* cannot make them right.
|
|
155
|
+
*/
|
|
156
|
+
export declare function fetchVerifiedArtifact(address: ArtifactAddress, locators: ArtifactLocators, options?: FetchVerifiedArtifactOptions): Promise<FetchVerifiedArtifactResult>;
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Artifact retrieval primitive — fetch by content address, verify, return bytes.
|
|
3
|
+
*
|
|
4
|
+
* One function fetches an artifact from the network legs an envelope names and
|
|
5
|
+
* hands back its bytes only after they hash to the recorded sha256, together
|
|
6
|
+
* with the provenance of the leg that produced them. A digest mismatch fails
|
|
7
|
+
* closed: the failure arm of the result union has no bytes on it at all.
|
|
8
|
+
*
|
|
9
|
+
* Keyless and filesystem-neutral by construction. The primitive itself neither
|
|
10
|
+
* accepts nor touches key material or the filesystem: it takes no store, no
|
|
11
|
+
* signer, no private key, and no path, and it performs no write of any kind.
|
|
12
|
+
* The one channel through which such material can reach a call is the caller's
|
|
13
|
+
* own `deps` seam closure — `acquire.ts` legitimately supplies one that holds a
|
|
14
|
+
* private key — and that closure is opaque here: it is invoked, never read,
|
|
15
|
+
* never logged, and never surfaced in provenance.
|
|
16
|
+
* `test/architecture/keyless-primitive.test.ts` holds the module graph to the
|
|
17
|
+
* same promise.
|
|
18
|
+
*
|
|
19
|
+
* Where the returned bytes land is the caller's decision. The daemon chain
|
|
20
|
+
* (`acquire.ts`) composes this with its byte cache; the CLI hands them to the
|
|
21
|
+
* operator; the MCP handler proxies to a daemon that runs it. None of that is
|
|
22
|
+
* this function's business.
|
|
23
|
+
*
|
|
24
|
+
* Issue #4179 (umbrella #2448 / journey spec B0c).
|
|
25
|
+
*/
|
|
26
|
+
import { createHash } from 'node:crypto';
|
|
27
|
+
import { buildArtifactUrl, fetchArtifactContent, } from './fetch-artifact.js';
|
|
28
|
+
import { classifyIpfsFetchFailure, fetchFromIpfs as defaultFetchFromIpfs } from './ipfs.js';
|
|
29
|
+
const DONATION_ARTIFACT_ENCODING = 'jinn.artifact.donation.v1';
|
|
30
|
+
/** Nothing that was learned tells us to stop; everything else does. */
|
|
31
|
+
const RETRYABLE_REASONS = new Set([
|
|
32
|
+
'timeout',
|
|
33
|
+
'unavailable',
|
|
34
|
+
]);
|
|
35
|
+
/**
|
|
36
|
+
* The one digest check in the system. Legs that already hold bytes — the byte
|
|
37
|
+
* cache, the self-store, a route resolver, the MCP daemon proxy — call this
|
|
38
|
+
* rather than writing `createHash('sha256')` out again.
|
|
39
|
+
*/
|
|
40
|
+
export function verifyArtifactDigest(expectedSha256, bytes) {
|
|
41
|
+
const actualSha256 = createHash('sha256').update(bytes).digest('hex');
|
|
42
|
+
return actualSha256 === expectedSha256 ? { ok: true } : { ok: false, actualSha256 };
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Log an IPFS read the retrieval is about to fall through, unless the gateway
|
|
46
|
+
* actually answered "not there" (#3441). A cap refusal is positive evidence the
|
|
47
|
+
* content exists, so it must not read like an ordinary miss.
|
|
48
|
+
*/
|
|
49
|
+
function warnIpfsFallThrough(subject, error) {
|
|
50
|
+
const classification = classifyIpfsFetchFailure(error);
|
|
51
|
+
if (classification === 'not-found')
|
|
52
|
+
return;
|
|
53
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
54
|
+
console.warn(`[corpus-read] IPFS source for ${subject} could not be used (${classification}), `
|
|
55
|
+
+ `falling through to the next source: ${detail}`);
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Decode a `jinn.artifact.donation.v1` payload into its raw bytes.
|
|
59
|
+
*
|
|
60
|
+
* Moved verbatim from `acquire.ts`. The `schemaVersion` check compares against
|
|
61
|
+
* the encoding constant — the published fixtures set both fields to that same
|
|
62
|
+
* string, so the comparison is load-bearing as written and is not "corrected"
|
|
63
|
+
* here.
|
|
64
|
+
*/
|
|
65
|
+
function decodeDonationArtifact(raw, expectedSha256) {
|
|
66
|
+
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
|
|
67
|
+
throw new Error('donation artifact payload is not an object');
|
|
68
|
+
}
|
|
69
|
+
const record = raw;
|
|
70
|
+
if (record['schemaVersion'] !== DONATION_ARTIFACT_ENCODING || record['encoding'] !== DONATION_ARTIFACT_ENCODING) {
|
|
71
|
+
throw new Error('donation artifact payload has unexpected encoding');
|
|
72
|
+
}
|
|
73
|
+
if (record['sha256'] !== expectedSha256) {
|
|
74
|
+
throw new Error('donation artifact sha256 does not match requested artifact');
|
|
75
|
+
}
|
|
76
|
+
if (typeof record['data'] !== 'string') {
|
|
77
|
+
throw new Error('donation artifact payload is missing base64 data');
|
|
78
|
+
}
|
|
79
|
+
return Buffer.from(record['data'], 'base64');
|
|
80
|
+
}
|
|
81
|
+
/** `fetchArtifactContent`'s vocabulary → the primitive's. */
|
|
82
|
+
function originReason(reason) {
|
|
83
|
+
return reason === 'network_error' ? 'unavailable' : reason;
|
|
84
|
+
}
|
|
85
|
+
/** `classifyIpfsFetchFailure`'s hyphenated vocabulary → the primitive's. */
|
|
86
|
+
function ipfsReason(classification) {
|
|
87
|
+
if (classification === 'too-large')
|
|
88
|
+
return 'too_large';
|
|
89
|
+
if (classification === 'not-found')
|
|
90
|
+
return 'not_found';
|
|
91
|
+
return 'unavailable';
|
|
92
|
+
}
|
|
93
|
+
function errorMessage(error) {
|
|
94
|
+
return error instanceof Error ? error.message : String(error);
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Fetch an artifact by content address and return its bytes only after they
|
|
98
|
+
* hash to that address.
|
|
99
|
+
*
|
|
100
|
+
* Legs run in the order the daemon chain has always used: the free donated IPFS
|
|
101
|
+
* mirror first, the operator's HTTP origin second. A leg with no locator is
|
|
102
|
+
* *skipped*, not failed. A digest mismatch on any leg fails the whole call and
|
|
103
|
+
* does not fall through — the bytes are wrong or hostile, and the next locator
|
|
104
|
+
* cannot make them right.
|
|
105
|
+
*/
|
|
106
|
+
export async function fetchVerifiedArtifact(address, locators, options) {
|
|
107
|
+
const { sha256, artifactType } = address;
|
|
108
|
+
const { sources = [], ipfsGatewayUrl, endpoint, envelopeCid, ownerSafe } = locators;
|
|
109
|
+
const fetchIpfs = options?.deps?.fetchFromIpfs ?? defaultFetchFromIpfs;
|
|
110
|
+
const fetchOrigin = options?.deps?.fetchArtifact ?? fetchArtifactContent;
|
|
111
|
+
const now = options?.deps?.now ?? (() => new Date().toISOString());
|
|
112
|
+
const attempts = [];
|
|
113
|
+
const admit = (source, sourceUri, bytes) => {
|
|
114
|
+
attempts.push({ leg: source, sourceUri, outcome: 'verified' });
|
|
115
|
+
return {
|
|
116
|
+
ok: true,
|
|
117
|
+
artifact: {
|
|
118
|
+
sha256,
|
|
119
|
+
bytes,
|
|
120
|
+
artifactType: artifactType ?? 'unknown',
|
|
121
|
+
provenance: {
|
|
122
|
+
source,
|
|
123
|
+
sourceUri,
|
|
124
|
+
digestAlgorithm: 'sha256',
|
|
125
|
+
fetchedAt: now(),
|
|
126
|
+
...(ownerSafe ? { sourceOperator: ownerSafe } : {}),
|
|
127
|
+
...(envelopeCid ? { envelopeCid } : {}),
|
|
128
|
+
attempts,
|
|
129
|
+
},
|
|
130
|
+
},
|
|
131
|
+
};
|
|
132
|
+
};
|
|
133
|
+
const refuse = (leg, sourceUri, actualSha256) => {
|
|
134
|
+
attempts.push({ leg, sourceUri, outcome: 'digest_mismatch' });
|
|
135
|
+
return {
|
|
136
|
+
ok: false,
|
|
137
|
+
sha256,
|
|
138
|
+
reason: 'digest_mismatch',
|
|
139
|
+
retryable: false,
|
|
140
|
+
message: `artifact ${sha256} from ${sourceUri} hashed to ${actualSha256}; `
|
|
141
|
+
+ 'the bytes were discarded',
|
|
142
|
+
attempts,
|
|
143
|
+
mismatch: {
|
|
144
|
+
expectedSha256: sha256,
|
|
145
|
+
actualSha256,
|
|
146
|
+
sourceUri,
|
|
147
|
+
...(ownerSafe ? { sourceOperator: ownerSafe } : {}),
|
|
148
|
+
},
|
|
149
|
+
};
|
|
150
|
+
};
|
|
151
|
+
// 1. Donated IPFS mirror — free and public, so it goes first.
|
|
152
|
+
const ipfsSource = sources.find((source) => source.kind === 'ipfs');
|
|
153
|
+
if (ipfsSource && ipfsGatewayUrl) {
|
|
154
|
+
const sourceUri = `ipfs://${ipfsSource.cid}`;
|
|
155
|
+
try {
|
|
156
|
+
const raw = await fetchIpfs(ipfsGatewayUrl, ipfsSource.cid);
|
|
157
|
+
const bytes = decodeDonationArtifact(raw, sha256);
|
|
158
|
+
const verified = verifyArtifactDigest(sha256, bytes);
|
|
159
|
+
if (!verified.ok)
|
|
160
|
+
return refuse('ipfs', sourceUri, verified.actualSha256);
|
|
161
|
+
return admit('ipfs', sourceUri, bytes);
|
|
162
|
+
}
|
|
163
|
+
catch (err) {
|
|
164
|
+
// A gateway failure or a malformed donation payload is opportunistic
|
|
165
|
+
// noise, not a verdict on the artifact: classify, warn unless it was
|
|
166
|
+
// proven absence, and let the next locator answer.
|
|
167
|
+
warnIpfsFallThrough(`donation artifact ${sha256}`, err);
|
|
168
|
+
attempts.push({
|
|
169
|
+
leg: 'ipfs',
|
|
170
|
+
sourceUri,
|
|
171
|
+
outcome: 'failed',
|
|
172
|
+
reason: ipfsReason(classifyIpfsFetchFailure(err)),
|
|
173
|
+
message: errorMessage(err),
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
else {
|
|
178
|
+
attempts.push({ leg: 'ipfs', sourceUri: '', outcome: 'skipped' });
|
|
179
|
+
}
|
|
180
|
+
// 2. Operator HTTP origin.
|
|
181
|
+
if (endpoint) {
|
|
182
|
+
const sourceUri = buildArtifactUrl(endpoint, sha256);
|
|
183
|
+
let outcome;
|
|
184
|
+
try {
|
|
185
|
+
outcome = await fetchOrigin(endpoint, sha256);
|
|
186
|
+
}
|
|
187
|
+
catch (err) {
|
|
188
|
+
outcome = { ok: false, reason: 'network_error', message: errorMessage(err) };
|
|
189
|
+
}
|
|
190
|
+
if (outcome.ok) {
|
|
191
|
+
const verified = verifyArtifactDigest(sha256, outcome.content);
|
|
192
|
+
if (!verified.ok)
|
|
193
|
+
return refuse('origin', sourceUri, verified.actualSha256);
|
|
194
|
+
return admit('origin', sourceUri, outcome.content);
|
|
195
|
+
}
|
|
196
|
+
attempts.push({
|
|
197
|
+
leg: 'origin',
|
|
198
|
+
sourceUri,
|
|
199
|
+
outcome: 'failed',
|
|
200
|
+
reason: originReason(outcome.reason),
|
|
201
|
+
...(outcome.message ? { message: outcome.message } : {}),
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
else {
|
|
205
|
+
attempts.push({ leg: 'origin', sourceUri: '', outcome: 'skipped' });
|
|
206
|
+
}
|
|
207
|
+
// Nothing verified. Absence is only claimed when every attempted leg said so:
|
|
208
|
+
// one inconclusive leg means we learned nothing about existence (#3441).
|
|
209
|
+
const failures = attempts.filter((attempt) => attempt.outcome === 'failed');
|
|
210
|
+
if (failures.length === 0) {
|
|
211
|
+
// Name why each leg was skipped rather than assuming both locators were
|
|
212
|
+
// absent: donated sources with no gateway to read them through is its own
|
|
213
|
+
// case, and reporting it as "no donated IPFS source" is simply wrong.
|
|
214
|
+
const missing = [
|
|
215
|
+
ipfsSource
|
|
216
|
+
? 'a donated IPFS source but no gateway URL to read it through'
|
|
217
|
+
: 'no donated IPFS source',
|
|
218
|
+
'no origin endpoint',
|
|
219
|
+
];
|
|
220
|
+
return {
|
|
221
|
+
ok: false,
|
|
222
|
+
sha256,
|
|
223
|
+
reason: 'no_locator',
|
|
224
|
+
retryable: false,
|
|
225
|
+
message: `artifact ${sha256} has no locator to try: ${missing.join(', and ')}`,
|
|
226
|
+
attempts,
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
// When more than one leg fails inconclusively, the last one wins. Legs run
|
|
230
|
+
// cheapest-first — the opportunistic donated mirror, then the operator's own
|
|
231
|
+
// origin — so the last leg to answer is the artifact's actual home, and its
|
|
232
|
+
// reason is the one the operator can act on.
|
|
233
|
+
const inconclusive = failures.filter((attempt) => attempt.reason !== 'not_found');
|
|
234
|
+
const reason = inconclusive.length > 0
|
|
235
|
+
? inconclusive[inconclusive.length - 1].reason
|
|
236
|
+
: 'not_found';
|
|
237
|
+
return {
|
|
238
|
+
ok: false,
|
|
239
|
+
sha256,
|
|
240
|
+
reason,
|
|
241
|
+
retryable: RETRYABLE_REASONS.has(reason),
|
|
242
|
+
message: `artifact ${sha256} could not be retrieved (${reason}) from `
|
|
243
|
+
+ failures.map((attempt) => `${attempt.sourceUri}: ${attempt.reason}`).join('; '),
|
|
244
|
+
attempts,
|
|
245
|
+
};
|
|
246
|
+
}
|
|
@@ -8,14 +8,59 @@
|
|
|
8
8
|
*
|
|
9
9
|
* Keys artifacts by sha256 (not IPFS CID) — artifacts no longer have IPFS
|
|
10
10
|
* CIDs (post jinn-mono-vy37.1.2).
|
|
11
|
+
*
|
|
12
|
+
* The endpoint is manifest-supplied and therefore attacker-controlled (#1901),
|
|
13
|
+
* so this fetch is both destination-restricted and resource-bounded *before*
|
|
14
|
+
* the hash check in `acquire.ts` ever runs:
|
|
15
|
+
*
|
|
16
|
+
* - every destination — the origin and each redirect hop — must pass
|
|
17
|
+
* `resolvePublicHttpDestination` (see `origin-guard.ts`), and the socket is
|
|
18
|
+
* pinned to the exact address that check approved, so a name whose DNS
|
|
19
|
+
* answer flips between validation and connect cannot escape the policy;
|
|
20
|
+
* - redirects are followed manually and capped;
|
|
21
|
+
* - one deadline bounds the whole chain, including the body read, so a
|
|
22
|
+
* stalled peer cannot hold a worker;
|
|
23
|
+
* - the body is streamed through a byte counter and abandoned the moment it
|
|
24
|
+
* exceeds the cap, so an oversized response is never fully buffered.
|
|
11
25
|
*/
|
|
26
|
+
import { type HostnameResolver } from './origin-guard.js';
|
|
27
|
+
import { type PinnedFetch } from './pinned-fetch.js';
|
|
12
28
|
export type AcquireResult = {
|
|
13
29
|
ok: true;
|
|
14
30
|
content: Buffer;
|
|
15
31
|
} | {
|
|
16
32
|
ok: false;
|
|
17
|
-
|
|
33
|
+
/**
|
|
34
|
+
* `blocked` — destination policy refused the origin or a redirect hop.
|
|
35
|
+
* `too_large` — the response exceeded the byte cap.
|
|
36
|
+
* `timeout` — the response stalled past the deadline.
|
|
37
|
+
*/
|
|
38
|
+
reason: 'not_found' | 'network_error' | 'blocked' | 'too_large' | 'timeout';
|
|
18
39
|
message?: string;
|
|
19
40
|
};
|
|
41
|
+
export interface FetchArtifactOptions {
|
|
42
|
+
/**
|
|
43
|
+
* Injection seam for tests. The default connects over `node:http(s)` with
|
|
44
|
+
* the socket pinned to the guard-approved address; a plain `fetch` fake is
|
|
45
|
+
* accepted here but does not pin, so production must not supply one.
|
|
46
|
+
*/
|
|
47
|
+
fetchImpl?: PinnedFetch;
|
|
48
|
+
/** Injection seam for tests; defaults to `dns.lookup(..., { all: true })`. */
|
|
49
|
+
resolveHostname?: HostnameResolver;
|
|
50
|
+
/**
|
|
51
|
+
* Permit loopback/private destinations, and skip address pinning with them.
|
|
52
|
+
* Off by default and set nowhere in-repo; it exists for an operator running
|
|
53
|
+
* against a local origin, because `operator.publicEndpoint` falls back to
|
|
54
|
+
* `http://localhost:<apiPort>` when unset. Scheme and credential checks
|
|
55
|
+
* still apply. Env: `JINN_CORPUS_ALLOW_PRIVATE_ORIGINS`.
|
|
56
|
+
*/
|
|
57
|
+
allowPrivateDestinations?: boolean;
|
|
58
|
+
/** Byte cap. Env: `JINN_CORPUS_ARTIFACT_MAX_BYTES`. */
|
|
59
|
+
maxBytes?: number;
|
|
60
|
+
/** Whole-operation deadline in ms; `0` disables. Env: `JINN_CORPUS_ARTIFACT_FETCH_TIMEOUT_MS`. */
|
|
61
|
+
timeoutMs?: number;
|
|
62
|
+
/** Redirect hop cap. Env: `JINN_CORPUS_ARTIFACT_MAX_REDIRECTS`. */
|
|
63
|
+
maxRedirects?: number;
|
|
64
|
+
}
|
|
20
65
|
export declare function buildArtifactUrl(endpoint: string, sha256: string): string;
|
|
21
|
-
export declare function fetchArtifactContent(endpoint: string, sha256: string): Promise<AcquireResult>;
|
|
66
|
+
export declare function fetchArtifactContent(endpoint: string, sha256: string, options?: FetchArtifactOptions): Promise<AcquireResult>;
|