@ariestools/aries-datalake-client 0.1.20 → 0.1.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +75 -1
- package/dist/browser/CompositePayloadsClient.d.ts +91 -0
- package/dist/browser/CompositePayloadsClient.d.ts.map +1 -0
- package/dist/browser/browser.d.ts +2 -0
- package/dist/browser/browser.d.ts.map +1 -1
- package/dist/browser/browser.mjs +253 -1
- package/dist/browser/browser.mjs.map +3 -3
- package/dist/node/CompositePayloadsClient.d.ts +91 -0
- package/dist/node/CompositePayloadsClient.d.ts.map +1 -0
- package/dist/node/LocalDatalakeClient.d.ts.map +1 -1
- package/dist/node/browser.d.ts +2 -0
- package/dist/node/browser.d.ts.map +1 -1
- package/dist/node/defaultDatalake.d.ts.map +1 -1
- package/dist/node/index.d.ts +2 -0
- package/dist/node/index.d.ts.map +1 -1
- package/dist/node/index.mjs +263 -6
- package/dist/node/index.mjs.map +3 -3
- package/package.json +16 -17
package/README.md
CHANGED
|
@@ -5,7 +5,7 @@ REST clients for the Aries datalake control and data planes, with Node-only loca
|
|
|
5
5
|
## Entry points
|
|
6
6
|
|
|
7
7
|
- `@ariestools/aries-datalake-client` is the Node entry point. It includes the REST clients plus local filesystem-backed clients, credentials, defaults, and `ARIES_HOME` helpers.
|
|
8
|
-
- `@ariestools/aries-datalake-client/browser` is the browser entry point. It exports
|
|
8
|
+
- `@ariestools/aries-datalake-client/browser` is the browser entry point. It exports REST clients, the composite client, their interfaces, the asynchronous token-supplier contract, and browser-safe datalake wire contracts.
|
|
9
9
|
|
|
10
10
|
Use an asynchronous token supplier that is invoked for each request. The application adapter may return a still-valid short-lived token from page memory or request a fresh wallet JWT through the XL1 browser wallet gateway:
|
|
11
11
|
|
|
@@ -37,6 +37,80 @@ const payloads = new RestPayloadsClient({
|
|
|
37
37
|
|
|
38
38
|
The supplier receives the exact control-plane audience or datalake id. Treat its return value as a sensitive, short-lived bearer token: cache it only in page memory when useful, and never persist it in browser storage or logs. The wallet gateway controls key access; the Aries HTTP services independently verify audience, signature, origin, scope, signer identity, and live ACL state.
|
|
39
39
|
|
|
40
|
+
## Multiple required datalakes
|
|
41
|
+
|
|
42
|
+
`CompositePayloadsClient` writes each target's eligible subset and verifies it by
|
|
43
|
+
reading the content back. Configure a complete S3 collection alongside a selected
|
|
44
|
+
Auto Drive collection using already authenticated `RestPayloadsClient` instances:
|
|
45
|
+
|
|
46
|
+
```ts
|
|
47
|
+
import { CompositePayloadsClient, type RestPayloadsClient } from '@ariestools/aries-datalake-client/browser'
|
|
48
|
+
import { asAnyPayload, PayloadBuilder } from '@xyo-network/sdk'
|
|
49
|
+
|
|
50
|
+
declare const autoDriveClient: RestPayloadsClient
|
|
51
|
+
declare const s3Client: RestPayloadsClient
|
|
52
|
+
|
|
53
|
+
const composite = new CompositePayloadsClient({
|
|
54
|
+
identify: async (payload) => {
|
|
55
|
+
const canonical = asAnyPayload(payload, true)
|
|
56
|
+
return {
|
|
57
|
+
hash: await PayloadBuilder.hash(canonical),
|
|
58
|
+
dataHash: await PayloadBuilder.dataHash(canonical),
|
|
59
|
+
}
|
|
60
|
+
},
|
|
61
|
+
targets: [
|
|
62
|
+
{
|
|
63
|
+
name: 'auto-drive',
|
|
64
|
+
client: autoDriveClient,
|
|
65
|
+
policy: {
|
|
66
|
+
mode: 'selected',
|
|
67
|
+
allowedSchemas: ['com.example.smallrecord'],
|
|
68
|
+
maxPayloadBytes: 4096, // Example deployment limit, not a provider default.
|
|
69
|
+
revision: 'v1',
|
|
70
|
+
},
|
|
71
|
+
},
|
|
72
|
+
{ name: 's3', client: s3Client, policy: { mode: 'all' } },
|
|
73
|
+
],
|
|
74
|
+
})
|
|
75
|
+
|
|
76
|
+
const result = await composite.insertWithReceipts([
|
|
77
|
+
{ schema: 'com.example.smallrecord', value: 1 },
|
|
78
|
+
{ schema: 'com.example.other', value: 2 },
|
|
79
|
+
])
|
|
80
|
+
const read = await composite.getWithReceipts(result.acknowledged.map(payload => payload._hash))
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
All targets are required for their eligible subset. A selected policy requires
|
|
84
|
+
an explicit allowlist and positive byte limit; an empty list selects nothing.
|
|
85
|
+
`disallowedSchemas` takes precedence. `schemaMaxPayloadBytes` may tighten the
|
|
86
|
+
global byte limit, and `isValid` may supply a trusted structural validator.
|
|
87
|
+
Sizes count UTF-8 JSON including client metadata and excluding storage metadata.
|
|
88
|
+
The receiving service must enforce its policy independently; this client does not
|
|
89
|
+
authorize permanent uploads or implement service quotas.
|
|
90
|
+
|
|
91
|
+
`insert(payloads)` returns the verified acknowledgment array, including duplicates;
|
|
92
|
+
`insertWithReceipts(payloads)` also returns per-target eligibility and outcomes.
|
|
93
|
+
A rejected expected-eligible item or failed required target throws
|
|
94
|
+
`CompositePayloadsWriteError`. Its `result` retains verified partial success and
|
|
95
|
+
sanitized receipts, not raw provider errors. A complete target's success does not
|
|
96
|
+
satisfy a different target's obligation. Callers own durable retry state and
|
|
97
|
+
policy-revision coordination; this client does not resume a failed operation.
|
|
98
|
+
|
|
99
|
+
`get(hashes)` and `getMany(hashes)` return verified payload arrays.
|
|
100
|
+
`getWithReceipts(hashes)` adds source attribution. Reads try targets in declared
|
|
101
|
+
order for unresolved hashes only. Provider errors or corrupt responses throw
|
|
102
|
+
`CompositePayloadsReadError`; they are not silently converted into misses.
|
|
103
|
+
Reads never copy content between targets. Supply canonical XYO `PayloadBuilder`
|
|
104
|
+
hashes through `identify`; the composite package itself adds no XYO runtime
|
|
105
|
+
dependency to browser consumers.
|
|
106
|
+
|
|
107
|
+
Only flat payload bodies are accepted. Validate and flatten known hydrated
|
|
108
|
+
protocol envelopes before insertion. There is no global sequence, clear, delete,
|
|
109
|
+
or usage API and no automatic registration into an XL1 connection. The
|
|
110
|
+
payload-array `insert`/`get` methods are intended for a protocol adapter; keep
|
|
111
|
+
the SDK's exact transaction-content acknowledgment check in that adapter.
|
|
112
|
+
Successful read-back is readable storage, not proof of provider network archival.
|
|
113
|
+
|
|
40
114
|
## Explicit public payload reads
|
|
41
115
|
|
|
42
116
|
Use `RestPublicPayloadsReader` when the provider has granted public viewer access
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import type { Payload, WithStorageMeta } from '@ariestools/aries-datalake-core/browser';
|
|
2
|
+
import type { PayloadsClient } from './PayloadsClient.ts';
|
|
3
|
+
/** Supply the canonical XYO PayloadBuilder hashes, without adding its runtime to browser clients. */
|
|
4
|
+
export interface CompositePayloadIdentity {
|
|
5
|
+
dataHash: string;
|
|
6
|
+
hash: string;
|
|
7
|
+
}
|
|
8
|
+
export interface CompositePayloadsPolicy {
|
|
9
|
+
/** Required for selected mode; an empty array selects nothing. */
|
|
10
|
+
allowedSchemas?: readonly string[];
|
|
11
|
+
disallowedSchemas?: readonly string[];
|
|
12
|
+
/** Trusted local validator; receiving services must independently enforce their policy. */
|
|
13
|
+
isValid?: (payload: Payload) => boolean;
|
|
14
|
+
/** Required for selected mode. Measured as UTF-8 JSON, before compression. */
|
|
15
|
+
maxPayloadBytes?: number;
|
|
16
|
+
/** Explicitly choose a complete collection or an allowlisted collection. */
|
|
17
|
+
mode: 'all' | 'selected';
|
|
18
|
+
revision?: string;
|
|
19
|
+
/** Per-schema limits can only tighten the overall payload limit. */
|
|
20
|
+
schemaMaxPayloadBytes?: Readonly<Record<string, number>>;
|
|
21
|
+
}
|
|
22
|
+
export interface CompositePayloadsTarget {
|
|
23
|
+
client: Pick<PayloadsClient, 'getMany' | 'insert'>;
|
|
24
|
+
/** A non-secret identifier, not an endpoint or credential. Also determines receipt provenance. */
|
|
25
|
+
name: string;
|
|
26
|
+
policy: CompositePayloadsPolicy;
|
|
27
|
+
}
|
|
28
|
+
export interface CompositePayloadsClientOptions {
|
|
29
|
+
/** Trusted canonical XYO hash implementation. Storage metadata must not affect either hash. */
|
|
30
|
+
identify: (payload: Payload) => Promise<CompositePayloadIdentity>;
|
|
31
|
+
/** Every target is required for its eligible subset. Array order is read priority. */
|
|
32
|
+
targets: readonly CompositePayloadsTarget[];
|
|
33
|
+
}
|
|
34
|
+
export interface CompositePayloadExclusion {
|
|
35
|
+
hash: string;
|
|
36
|
+
reason: 'schema' | 'size' | 'validation';
|
|
37
|
+
}
|
|
38
|
+
export interface CompositeWriteReceipt {
|
|
39
|
+
acknowledged: string[];
|
|
40
|
+
eligible: string[];
|
|
41
|
+
error?: 'provider_error' | 'invalid_response' | 'policy_rejected' | 'incomplete_readback';
|
|
42
|
+
excluded: CompositePayloadExclusion[];
|
|
43
|
+
policyRevision?: string;
|
|
44
|
+
rejected: string[];
|
|
45
|
+
status: 'complete' | 'excluded' | 'failed';
|
|
46
|
+
target: string;
|
|
47
|
+
}
|
|
48
|
+
export interface CompositeWriteResult {
|
|
49
|
+
/** Verified union, including previously stored duplicates. Not an aggregate success on an error. */
|
|
50
|
+
acknowledged: WithStorageMeta<Payload>[];
|
|
51
|
+
receipts: CompositeWriteReceipt[];
|
|
52
|
+
}
|
|
53
|
+
export declare class CompositePayloadsWriteError extends Error {
|
|
54
|
+
readonly result: CompositeWriteResult;
|
|
55
|
+
constructor(result: CompositeWriteResult);
|
|
56
|
+
}
|
|
57
|
+
export interface CompositeReadReceipt {
|
|
58
|
+
error?: 'provider_error' | 'invalid_response';
|
|
59
|
+
found: string[];
|
|
60
|
+
requested: string[];
|
|
61
|
+
target: string;
|
|
62
|
+
}
|
|
63
|
+
export interface CompositeReadResult {
|
|
64
|
+
payloads: WithStorageMeta<Payload>[];
|
|
65
|
+
receipts: CompositeReadReceipt[];
|
|
66
|
+
sources: Record<string, string>;
|
|
67
|
+
}
|
|
68
|
+
export declare class CompositePayloadsReadError extends Error {
|
|
69
|
+
readonly result: CompositeReadResult;
|
|
70
|
+
constructor(result: CompositeReadResult);
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Required-subset writes and verified hash reads across named datalakes.
|
|
74
|
+
* Accepts flat payload bodies only. Callers validate and flatten protocol envelopes first.
|
|
75
|
+
* Deliberately has no global next/clear/delete/usage: store-local sequences and lifecycles differ.
|
|
76
|
+
*/
|
|
77
|
+
export declare class CompositePayloadsClient {
|
|
78
|
+
private readonly identify;
|
|
79
|
+
private readonly targets;
|
|
80
|
+
constructor(options: CompositePayloadsClientOptions);
|
|
81
|
+
/** Missing hashes fall through in target order. Provider errors and corruption fail explicitly. */
|
|
82
|
+
get(hashes: string[]): Promise<WithStorageMeta<Payload>[]>;
|
|
83
|
+
getMany(hashes: string[]): Promise<WithStorageMeta<Payload>[]>;
|
|
84
|
+
getWithReceipts(hashes: readonly string[]): Promise<CompositeReadResult>;
|
|
85
|
+
/** Payload-array acknowledgment suitable for consumers of insert/get method contracts. */
|
|
86
|
+
insert(payloads: Payload[]): Promise<WithStorageMeta<Payload>[]>;
|
|
87
|
+
insertWithReceipts(payloads: readonly Payload[]): Promise<CompositeWriteResult>;
|
|
88
|
+
private verify;
|
|
89
|
+
private writeTarget;
|
|
90
|
+
}
|
|
91
|
+
//# sourceMappingURL=CompositePayloadsClient.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"CompositePayloadsClient.d.ts","sourceRoot":"","sources":["../../src/CompositePayloadsClient.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,yCAAyC,CAAA;AAEvF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAA;AAEzD,qGAAqG;AACrG,MAAM,WAAW,wBAAwB;IACvC,QAAQ,EAAE,MAAM,CAAA;IAChB,IAAI,EAAE,MAAM,CAAA;CACb;AAED,MAAM,WAAW,uBAAuB;IACtC,kEAAkE;IAClE,cAAc,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAClC,iBAAiB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IACrC,2FAA2F;IAC3F,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,OAAO,CAAA;IACvC,8EAA8E;IAC9E,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,4EAA4E;IAC5E,IAAI,EAAE,KAAK,GAAG,UAAU,CAAA;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,oEAAoE;IACpE,qBAAqB,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;CACzD;AAED,MAAM,WAAW,uBAAuB;IACtC,MAAM,EAAE,IAAI,CAAC,cAAc,EAAE,SAAS,GAAG,QAAQ,CAAC,CAAA;IAClD,kGAAkG;IAClG,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,uBAAuB,CAAA;CAChC;AAED,MAAM,WAAW,8BAA8B;IAC7C,+FAA+F;IAC/F,QAAQ,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,wBAAwB,CAAC,CAAA;IACjE,sFAAsF;IACtF,OAAO,EAAE,SAAS,uBAAuB,EAAE,CAAA;CAC5C;AAED,MAAM,WAAW,yBAAyB;IACxC,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,QAAQ,GAAG,MAAM,GAAG,YAAY,CAAA;CACzC;AAED,MAAM,WAAW,qBAAqB;IACpC,YAAY,EAAE,MAAM,EAAE,CAAA;IACtB,QAAQ,EAAE,MAAM,EAAE,CAAA;IAClB,KAAK,CAAC,EAAE,gBAAgB,GAAG,kBAAkB,GAAG,iBAAiB,GAAG,qBAAqB,CAAA;IACzF,QAAQ,EAAE,yBAAyB,EAAE,CAAA;IACrC,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,QAAQ,EAAE,MAAM,EAAE,CAAA;IAClB,MAAM,EAAE,UAAU,GAAG,UAAU,GAAG,QAAQ,CAAA;IAC1C,MAAM,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,oBAAoB;IACnC,oGAAoG;IACpG,YAAY,EAAE,eAAe,CAAC,OAAO,CAAC,EAAE,CAAA;IACxC,QAAQ,EAAE,qBAAqB,EAAE,CAAA;CAClC;AAED,qBAAa,2BAA4B,SAAQ,KAAK;IACpD,QAAQ,CAAC,MAAM,EAAE,oBAAoB,CAAA;gBAEzB,MAAM,EAAE,oBAAoB;CAKzC;AAED,MAAM,WAAW,oBAAoB;IACnC,KAAK,CAAC,EAAE,gBAAgB,GAAG,kBAAkB,CAAA;IAC7C,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,SAAS,EAAE,MAAM,EAAE,CAAA;IACnB,MAAM,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,eAAe,CAAC,OAAO,CAAC,EAAE,CAAA;IACpC,QAAQ,EAAE,oBAAoB,EAAE,CAAA;IAChC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAChC;AAED,qBAAa,0BAA2B,SAAQ,KAAK;IACnD,QAAQ,CAAC,MAAM,EAAE,mBAAmB,CAAA;gBAExB,MAAM,EAAE,mBAAmB;CAKxC;AAcD;;;;GAIG;AACH,qBAAa,uBAAuB;IAClC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA4C;IACrE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoC;gBAEhD,OAAO,EAAE,8BAA8B;IAcnD,mGAAmG;IAC7F,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;IAI1D,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;IAI9D,eAAe,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,mBAAmB,CAAC;IAqC9E,0FAA0F;IACpF,MAAM,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;IAIhE,kBAAkB,CAAC,QAAQ,EAAE,SAAS,OAAO,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC;YA8BvE,MAAM;YAmBN,WAAW;CA8C1B"}
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
export type { AuthTokenRequest, AuthTokenSource, AuthTokenSupplier, } from './AuthToken.ts';
|
|
2
|
+
export type { CompositePayloadExclusion, CompositePayloadIdentity, CompositePayloadsClientOptions, CompositePayloadsPolicy, CompositePayloadsTarget, CompositeReadReceipt, CompositeReadResult, CompositeWriteReceipt, CompositeWriteResult, } from './CompositePayloadsClient.ts';
|
|
3
|
+
export { CompositePayloadsClient, CompositePayloadsReadError, CompositePayloadsWriteError, } from './CompositePayloadsClient.ts';
|
|
2
4
|
export type { CreateDatalakeRequest, DatalakeClient, GrantRequest, RevokeRequest, TokenRequest, } from './DatalakeClient.ts';
|
|
3
5
|
export type { DatalakeUsageReport, PayloadsAuthTokenSource, PayloadsClient, RestPayloadsClientOptions, } from './PayloadsClient.ts';
|
|
4
6
|
export { RestPayloadsClient } from './PayloadsClient.ts';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser.d.ts","sourceRoot":"","sources":["../../src/browser.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,gBAAgB,EAAE,eAAe,EAAE,iBAAiB,GACrD,MAAM,gBAAgB,CAAA;AACvB,YAAY,EACV,qBAAqB,EACrB,cAAc,EACd,YAAY,EACZ,aAAa,EACb,YAAY,GACb,MAAM,qBAAqB,CAAA;AAC5B,YAAY,EACV,mBAAmB,EACnB,uBAAuB,EACvB,cAAc,EACd,yBAAyB,GAC1B,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AACxD,YAAY,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAA;AACxE,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAA;AAC5D,YAAY,EAAE,+BAA+B,EAAE,6BAA6B,EAAE,MAAM,+BAA+B,CAAA;AACnH,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAA;AACxE,cAAc,yCAAyC,CAAA"}
|
|
1
|
+
{"version":3,"file":"browser.d.ts","sourceRoot":"","sources":["../../src/browser.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,gBAAgB,EAAE,eAAe,EAAE,iBAAiB,GACrD,MAAM,gBAAgB,CAAA;AACvB,YAAY,EACV,yBAAyB,EACzB,wBAAwB,EACxB,8BAA8B,EAC9B,uBAAuB,EACvB,uBAAuB,EACvB,oBAAoB,EACpB,mBAAmB,EACnB,qBAAqB,EACrB,oBAAoB,GACrB,MAAM,8BAA8B,CAAA;AACrC,OAAO,EACL,uBAAuB,EAAE,0BAA0B,EAAE,2BAA2B,GACjF,MAAM,8BAA8B,CAAA;AACrC,YAAY,EACV,qBAAqB,EACrB,cAAc,EACd,YAAY,EACZ,aAAa,EACb,YAAY,GACb,MAAM,qBAAqB,CAAA;AAC5B,YAAY,EACV,mBAAmB,EACnB,uBAAuB,EACvB,cAAc,EACd,yBAAyB,GAC1B,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AACxD,YAAY,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAA;AACxE,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAA;AAC5D,YAAY,EAAE,+BAA+B,EAAE,6BAA6B,EAAE,MAAM,+BAA+B,CAAA;AACnH,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAA;AACxE,cAAc,yCAAyC,CAAA"}
|
package/dist/browser/browser.mjs
CHANGED
|
@@ -1,3 +1,252 @@
|
|
|
1
|
+
// src/CompositePayloadsClient.ts
|
|
2
|
+
var CompositePayloadsWriteError = class extends Error {
|
|
3
|
+
result;
|
|
4
|
+
constructor(result) {
|
|
5
|
+
super("Required datalake writes did not complete");
|
|
6
|
+
this.name = "CompositePayloadsWriteError";
|
|
7
|
+
this.result = result;
|
|
8
|
+
}
|
|
9
|
+
};
|
|
10
|
+
var CompositePayloadsReadError = class extends Error {
|
|
11
|
+
result;
|
|
12
|
+
constructor(result) {
|
|
13
|
+
super("Datalake read could not be verified");
|
|
14
|
+
this.name = "CompositePayloadsReadError";
|
|
15
|
+
this.result = result;
|
|
16
|
+
}
|
|
17
|
+
};
|
|
18
|
+
var CompositePayloadsClient = class {
|
|
19
|
+
identify;
|
|
20
|
+
targets;
|
|
21
|
+
constructor(options) {
|
|
22
|
+
if (typeof options.identify !== "function") throw new TypeError("Canonical payload identity is required");
|
|
23
|
+
if (options.targets.length === 0) throw new TypeError("At least one datalake target is required");
|
|
24
|
+
const names = /* @__PURE__ */ new Set();
|
|
25
|
+
this.targets = options.targets.map((target) => {
|
|
26
|
+
if (!/^[a-z0-9][a-z0-9._-]{0,63}$/i.test(target.name) || names.has(target.name)) {
|
|
27
|
+
throw new TypeError("Datalake target names must be unique non-secret identifiers");
|
|
28
|
+
}
|
|
29
|
+
names.add(target.name);
|
|
30
|
+
return { ...target, policy: snapshotPolicy(target.policy) };
|
|
31
|
+
});
|
|
32
|
+
this.identify = options.identify;
|
|
33
|
+
}
|
|
34
|
+
/** Missing hashes fall through in target order. Provider errors and corruption fail explicitly. */
|
|
35
|
+
async get(hashes) {
|
|
36
|
+
return (await this.getWithReceipts(hashes)).payloads;
|
|
37
|
+
}
|
|
38
|
+
async getMany(hashes) {
|
|
39
|
+
return await this.get(hashes);
|
|
40
|
+
}
|
|
41
|
+
async getWithReceipts(hashes) {
|
|
42
|
+
const requested = [...new Set(hashes)];
|
|
43
|
+
if (requested.some((hash) => typeof hash !== "string" || hash.length === 0)) throw new TypeError("Hashes must be non-empty strings");
|
|
44
|
+
const found = /* @__PURE__ */ new Map();
|
|
45
|
+
const result = {
|
|
46
|
+
payloads: [],
|
|
47
|
+
receipts: [],
|
|
48
|
+
sources: {}
|
|
49
|
+
};
|
|
50
|
+
for (const target of this.targets) {
|
|
51
|
+
const remaining = requested.filter((hash) => !found.has(hash));
|
|
52
|
+
if (remaining.length === 0) break;
|
|
53
|
+
const receipt = {
|
|
54
|
+
target: target.name,
|
|
55
|
+
requested: remaining,
|
|
56
|
+
found: []
|
|
57
|
+
};
|
|
58
|
+
result.receipts.push(receipt);
|
|
59
|
+
let response;
|
|
60
|
+
try {
|
|
61
|
+
response = await target.client.getMany([...remaining]);
|
|
62
|
+
} catch {
|
|
63
|
+
receipt.error = "provider_error";
|
|
64
|
+
throw new CompositePayloadsReadError(result);
|
|
65
|
+
}
|
|
66
|
+
try {
|
|
67
|
+
const verified = await this.verify(response, new Set(remaining));
|
|
68
|
+
for (const payload of verified.values()) {
|
|
69
|
+
found.set(payload._hash, payload);
|
|
70
|
+
result.sources[payload._hash] = target.name;
|
|
71
|
+
receipt.found.push(payload._hash);
|
|
72
|
+
}
|
|
73
|
+
result.payloads = requested.flatMap((hash) => found.has(hash) ? [found.get(hash)] : []);
|
|
74
|
+
} catch {
|
|
75
|
+
receipt.error = "invalid_response";
|
|
76
|
+
throw new CompositePayloadsReadError(result);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
return result;
|
|
80
|
+
}
|
|
81
|
+
/** Payload-array acknowledgment suitable for consumers of insert/get method contracts. */
|
|
82
|
+
async insert(payloads) {
|
|
83
|
+
return (await this.insertWithReceipts(payloads)).acknowledged;
|
|
84
|
+
}
|
|
85
|
+
async insertWithReceipts(payloads) {
|
|
86
|
+
const snapshots = payloads.map(snapshotPayload);
|
|
87
|
+
const prepared = [];
|
|
88
|
+
const seen = /* @__PURE__ */ new Set();
|
|
89
|
+
const encoder = new TextEncoder();
|
|
90
|
+
for (const payload of snapshots) {
|
|
91
|
+
const { hash } = await this.identify(payload);
|
|
92
|
+
requireHash(hash);
|
|
93
|
+
if (!seen.has(hash)) {
|
|
94
|
+
prepared.push({
|
|
95
|
+
payload,
|
|
96
|
+
hash,
|
|
97
|
+
bytes: encoder.encode(JSON.stringify(payload)).byteLength
|
|
98
|
+
});
|
|
99
|
+
seen.add(hash);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
const writes = this.targets.map((target) => prepareWrite(target, prepared));
|
|
103
|
+
const outcomes = await Promise.all(writes.map((write) => this.writeTarget(write)));
|
|
104
|
+
const acknowledged = /* @__PURE__ */ new Map();
|
|
105
|
+
for (const outcome of outcomes) {
|
|
106
|
+
for (const payload of outcome.acknowledged) acknowledged.set(payload._hash, payload);
|
|
107
|
+
}
|
|
108
|
+
const result = {
|
|
109
|
+
acknowledged: prepared.flatMap(({ hash }) => acknowledged.has(hash) ? [acknowledged.get(hash)] : []),
|
|
110
|
+
receipts: writes.map((write) => write.receipt)
|
|
111
|
+
};
|
|
112
|
+
if (result.receipts.some((receipt) => receipt.status === "failed")) throw new CompositePayloadsWriteError(result);
|
|
113
|
+
return result;
|
|
114
|
+
}
|
|
115
|
+
async verify(payloads, expected) {
|
|
116
|
+
if (!Array.isArray(payloads)) throw new TypeError("Expected payload array");
|
|
117
|
+
const snapshots = payloads.map((payload) => {
|
|
118
|
+
if (typeof payload?._sequence !== "string" || payload._sequence.length === 0) throw new TypeError("Missing storage sequence");
|
|
119
|
+
return {
|
|
120
|
+
...snapshotPayload(payload),
|
|
121
|
+
_hash: payload._hash,
|
|
122
|
+
_dataHash: payload._dataHash,
|
|
123
|
+
_sequence: payload._sequence
|
|
124
|
+
};
|
|
125
|
+
});
|
|
126
|
+
const verified = /* @__PURE__ */ new Map();
|
|
127
|
+
for (const payload of snapshots) {
|
|
128
|
+
const identity = await this.identify(snapshotPayload(payload));
|
|
129
|
+
if (identity.hash !== payload._hash || identity.dataHash !== payload._dataHash || !expected.has(identity.hash)) {
|
|
130
|
+
throw new TypeError("Unexpected payload identity");
|
|
131
|
+
}
|
|
132
|
+
verified.set(identity.hash, payload);
|
|
133
|
+
}
|
|
134
|
+
return verified;
|
|
135
|
+
}
|
|
136
|
+
async writeTarget({
|
|
137
|
+
target,
|
|
138
|
+
payloads,
|
|
139
|
+
receipt
|
|
140
|
+
}) {
|
|
141
|
+
const result = { acknowledged: [], receipts: [receipt] };
|
|
142
|
+
if (payloads.length === 0) return result;
|
|
143
|
+
let response;
|
|
144
|
+
try {
|
|
145
|
+
response = await target.client.insert(payloads.map((item) => snapshotPayload(item.payload)));
|
|
146
|
+
} catch {
|
|
147
|
+
receipt.status = "failed";
|
|
148
|
+
receipt.error = "provider_error";
|
|
149
|
+
return result;
|
|
150
|
+
}
|
|
151
|
+
try {
|
|
152
|
+
await this.verify(response.inserted, new Set(receipt.eligible));
|
|
153
|
+
if (!Array.isArray(response.summary.rejected) || response.summary.rejected.some((hash) => !receipt.eligible.includes(hash))) {
|
|
154
|
+
throw new TypeError("Unexpected rejected hashes");
|
|
155
|
+
}
|
|
156
|
+
receipt.rejected = [...new Set(response.summary.rejected)];
|
|
157
|
+
} catch {
|
|
158
|
+
receipt.status = "failed";
|
|
159
|
+
receipt.error = "invalid_response";
|
|
160
|
+
return result;
|
|
161
|
+
}
|
|
162
|
+
let readback;
|
|
163
|
+
try {
|
|
164
|
+
readback = await target.client.getMany([...receipt.eligible]);
|
|
165
|
+
} catch {
|
|
166
|
+
receipt.status = "failed";
|
|
167
|
+
receipt.error = "provider_error";
|
|
168
|
+
return result;
|
|
169
|
+
}
|
|
170
|
+
try {
|
|
171
|
+
const verified = await this.verify(readback, new Set(receipt.eligible));
|
|
172
|
+
result.acknowledged = receipt.eligible.flatMap((hash) => verified.has(hash) ? [verified.get(hash)] : []);
|
|
173
|
+
receipt.acknowledged = result.acknowledged.map((payload) => payload._hash);
|
|
174
|
+
if (receipt.rejected.length > 0 || verified.size !== receipt.eligible.length) {
|
|
175
|
+
receipt.status = "failed";
|
|
176
|
+
receipt.error = receipt.rejected.length > 0 ? "policy_rejected" : "incomplete_readback";
|
|
177
|
+
}
|
|
178
|
+
} catch {
|
|
179
|
+
receipt.status = "failed";
|
|
180
|
+
receipt.error = "invalid_response";
|
|
181
|
+
}
|
|
182
|
+
return result;
|
|
183
|
+
}
|
|
184
|
+
};
|
|
185
|
+
function snapshotPolicy(policy) {
|
|
186
|
+
if (!policy || policy.mode !== "all" && policy.mode !== "selected") throw new TypeError("Explicit datalake policy is required");
|
|
187
|
+
if (policy.mode === "selected" && (!Array.isArray(policy.allowedSchemas) || policy.maxPayloadBytes === void 0)) {
|
|
188
|
+
throw new TypeError("Selected datalakes require an allowlist and byte limit");
|
|
189
|
+
}
|
|
190
|
+
for (const schemas of [policy.allowedSchemas, policy.disallowedSchemas]) {
|
|
191
|
+
if (schemas !== void 0 && (!Array.isArray(schemas) || schemas.some((schema) => typeof schema !== "string" || schema.length === 0))) {
|
|
192
|
+
throw new TypeError("Schema lists must contain non-empty identifiers");
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
for (const limit of [policy.maxPayloadBytes, ...Object.values(policy.schemaMaxPayloadBytes ?? {})]) {
|
|
196
|
+
if (limit !== void 0 && (!Number.isSafeInteger(limit) || limit <= 0)) throw new RangeError("Payload byte limits must be positive safe integers");
|
|
197
|
+
}
|
|
198
|
+
return {
|
|
199
|
+
...policy,
|
|
200
|
+
allowedSchemas: policy.allowedSchemas === void 0 ? void 0 : [...policy.allowedSchemas],
|
|
201
|
+
disallowedSchemas: policy.disallowedSchemas === void 0 ? void 0 : [...policy.disallowedSchemas],
|
|
202
|
+
schemaMaxPayloadBytes: { ...policy.schemaMaxPayloadBytes }
|
|
203
|
+
};
|
|
204
|
+
}
|
|
205
|
+
function prepareWrite(target, payloads) {
|
|
206
|
+
const selected = [];
|
|
207
|
+
const excluded = [];
|
|
208
|
+
for (const item of payloads) {
|
|
209
|
+
const reason = exclusionReason(target.policy, item);
|
|
210
|
+
if (reason) excluded.push({ hash: item.hash, reason });
|
|
211
|
+
else selected.push(item);
|
|
212
|
+
}
|
|
213
|
+
return {
|
|
214
|
+
target,
|
|
215
|
+
payloads: selected,
|
|
216
|
+
receipt: {
|
|
217
|
+
target: target.name,
|
|
218
|
+
policyRevision: target.policy.revision,
|
|
219
|
+
eligible: selected.map((item) => item.hash),
|
|
220
|
+
excluded,
|
|
221
|
+
acknowledged: [],
|
|
222
|
+
rejected: [],
|
|
223
|
+
status: selected.length > 0 ? "complete" : "excluded"
|
|
224
|
+
}
|
|
225
|
+
};
|
|
226
|
+
}
|
|
227
|
+
function exclusionReason(policy, item) {
|
|
228
|
+
const { schema } = item.payload;
|
|
229
|
+
if (policy.allowedSchemas !== void 0 && !policy.allowedSchemas.includes(schema) || policy.disallowedSchemas?.includes(schema)) return "schema";
|
|
230
|
+
const schemaLimit = policy.schemaMaxPayloadBytes && Object.hasOwn(policy.schemaMaxPayloadBytes, schema) ? policy.schemaMaxPayloadBytes[schema] : void 0;
|
|
231
|
+
const limit = Math.min(policy.maxPayloadBytes ?? Infinity, schemaLimit ?? Infinity);
|
|
232
|
+
if (item.bytes > limit) return "size";
|
|
233
|
+
if (policy.isValid && !policy.isValid(snapshotPayload(item.payload))) return "validation";
|
|
234
|
+
return void 0;
|
|
235
|
+
}
|
|
236
|
+
function snapshotPayload(payload) {
|
|
237
|
+
if (!payload || typeof payload !== "object" || Array.isArray(payload) || typeof payload.schema !== "string" || payload.schema.length === 0) {
|
|
238
|
+
throw new TypeError("Expected flat schema-bearing payloads");
|
|
239
|
+
}
|
|
240
|
+
const stripped = Object.fromEntries(Object.entries(payload).filter(([key]) => !key.startsWith("_")));
|
|
241
|
+
const serialized = JSON.stringify(stripped);
|
|
242
|
+
const copy = JSON.parse(serialized);
|
|
243
|
+
if (copy.schema !== payload.schema) throw new TypeError("Payload serialization changed its schema");
|
|
244
|
+
return copy;
|
|
245
|
+
}
|
|
246
|
+
function requireHash(hash) {
|
|
247
|
+
if (typeof hash !== "string" || hash.length === 0) throw new TypeError("Canonical payload identity returned an invalid hash");
|
|
248
|
+
}
|
|
249
|
+
|
|
1
250
|
// src/PayloadsClient.ts
|
|
2
251
|
import {
|
|
3
252
|
DATALAKE_HEADER_DUPLICATES,
|
|
@@ -136,7 +385,7 @@ var RestPayloadsClient = class {
|
|
|
136
385
|
};
|
|
137
386
|
function readInsertSummary(headers) {
|
|
138
387
|
const duplicatesRaw = headers.get(DATALAKE_HEADER_DUPLICATES);
|
|
139
|
-
const duplicates = duplicatesRaw === null ? 0 :
|
|
388
|
+
const duplicates = duplicatesRaw === null ? 0 : Math.trunc(Number(duplicatesRaw));
|
|
140
389
|
const rejectedRaw = headers.get(DATALAKE_HEADER_REJECTED);
|
|
141
390
|
const rejected = rejectedRaw === null || rejectedRaw.length === 0 ? [] : rejectedRaw.split(",").map((value) => value.trim()).filter((value) => value.length > 0);
|
|
142
391
|
return {
|
|
@@ -320,6 +569,9 @@ function isStoredPayload(value) {
|
|
|
320
569
|
// src/browser.ts
|
|
321
570
|
export * from "@ariestools/aries-datalake-core/browser";
|
|
322
571
|
export {
|
|
572
|
+
CompositePayloadsClient,
|
|
573
|
+
CompositePayloadsReadError,
|
|
574
|
+
CompositePayloadsWriteError,
|
|
323
575
|
RestDatalakeClient,
|
|
324
576
|
RestPayloadsClient,
|
|
325
577
|
RestPublicPayloadsReader
|