@bandeira-tech/b3nd-save 0.8.1 → 0.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/clients/save-client.d.ts +63 -29
- package/esm/clients/save-client.d.ts.map +1 -1
- package/esm/clients/save-client.js +148 -68
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/mod.d.ts +1 -1
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/0.25.0/mod.d.ts.map +1 -0
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/client-console/client.d.ts.map +1 -1
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/encoding/encoding.d.ts.map +1 -1
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/encrypt/mod.d.ts +1 -0
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/0.25.0/src/encrypt/mod.d.ts.map +1 -0
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/encrypt/mod.js +1 -0
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/0.25.0/src/encrypt/utils.d.ts +6 -0
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/0.25.0/src/encrypt/utils.d.ts.map +1 -0
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/0.25.0/src/encrypt/utils.js +12 -0
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/functional-client/functional-client.d.ts.map +1 -1
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/match-pattern/match-pattern.d.ts.map +1 -1
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/network/decorators.d.ts.map +1 -1
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/network/network.d.ts.map +1 -1
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/network/peer.d.ts.map +1 -1
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/network/policies/flood.d.ts.map +1 -1
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/network/policies/path-vector.d.ts +3 -1
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/0.25.0/src/network/policies/path-vector.d.ts.map +1 -0
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/network/policies/path-vector.js +3 -1
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/network/policies/tell-and-read.d.ts.map +1 -1
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/network/types.d.ts.map +1 -1
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/observe-emitter/observe-emitter.d.ts.map +1 -1
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/rig/connection.d.ts +13 -5
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/0.25.0/src/rig/connection.d.ts.map +1 -0
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/rig/connection.js +1 -1
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/rig/events.d.ts.map +1 -1
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/rig/hooks.d.ts.map +1 -1
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/rig/identity.d.ts.map +1 -1
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/rig/operation-handle.d.ts.map +1 -1
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/rig/reactions.d.ts.map +1 -1
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/rig/rig.d.ts +3 -0
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/0.25.0/src/rig/rig.d.ts.map +1 -0
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/rig/rig.js +46 -4
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/rig/types.d.ts +4 -4
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/0.25.0/src/rig/types.d.ts.map +1 -0
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/types/types.d.ts +119 -80
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/0.25.0/src/types/types.d.ts.map +1 -0
- package/esm/dispatch.d.ts +116 -3
- package/esm/dispatch.d.ts.map +1 -1
- package/esm/dispatch.js +381 -5
- package/esm/elasticsearch/store.d.ts +124 -25
- package/esm/elasticsearch/store.d.ts.map +1 -1
- package/esm/elasticsearch/store.js +451 -92
- package/esm/entity-store.d.ts +1 -1
- package/esm/errors.d.ts +1 -1
- package/esm/errors.js +1 -1
- package/esm/fs/mod.d.ts +26 -0
- package/esm/fs/mod.d.ts.map +1 -1
- package/esm/fs/store.d.ts +100 -23
- package/esm/fs/store.d.ts.map +1 -1
- package/esm/fs/store.js +250 -66
- package/esm/glob.d.ts +114 -0
- package/esm/glob.d.ts.map +1 -0
- package/esm/glob.js +352 -0
- package/esm/identifiers.d.ts +18 -0
- package/esm/identifiers.d.ts.map +1 -0
- package/esm/identifiers.js +19 -0
- package/esm/indexeddb/fields.d.ts +32 -0
- package/esm/indexeddb/fields.d.ts.map +1 -0
- package/esm/indexeddb/fields.js +51 -0
- package/esm/indexeddb/store.d.ts +72 -17
- package/esm/indexeddb/store.d.ts.map +1 -1
- package/esm/indexeddb/store.js +362 -106
- package/esm/ipfs/mod.d.ts +19 -0
- package/esm/ipfs/mod.d.ts.map +1 -1
- package/esm/ipfs/mod.js +19 -0
- package/esm/ipfs/store.d.ts +82 -21
- package/esm/ipfs/store.d.ts.map +1 -1
- package/esm/ipfs/store.js +269 -68
- package/esm/json-fields.d.ts +61 -0
- package/esm/json-fields.d.ts.map +1 -0
- package/esm/json-fields.js +121 -0
- package/esm/localstorage/store.d.ts +89 -21
- package/esm/localstorage/store.d.ts.map +1 -1
- package/esm/localstorage/store.js +247 -54
- package/esm/memory/store.d.ts +35 -4
- package/esm/memory/store.d.ts.map +1 -1
- package/esm/memory/store.js +69 -47
- package/esm/mod.d.ts +2 -0
- package/esm/mod.d.ts.map +1 -1
- package/esm/mod.js +1 -0
- package/esm/mongo/fields.d.ts +53 -0
- package/esm/mongo/fields.d.ts.map +1 -0
- package/esm/mongo/fields.js +71 -0
- package/esm/mongo/mod.d.ts +19 -4
- package/esm/mongo/mod.d.ts.map +1 -1
- package/esm/mongo/mod.js +4 -2
- package/esm/mongo/store.d.ts +114 -25
- package/esm/mongo/store.d.ts.map +1 -1
- package/esm/mongo/store.js +355 -102
- package/esm/postgres/store.d.ts +50 -1
- package/esm/postgres/store.d.ts.map +1 -1
- package/esm/postgres/store.js +133 -13
- package/esm/read.d.ts +54 -7
- package/esm/read.d.ts.map +1 -1
- package/esm/read.js +141 -16
- package/esm/s3/store.d.ts +125 -23
- package/esm/s3/store.d.ts.map +1 -1
- package/esm/s3/store.js +313 -57
- package/esm/sqlite/columns.d.ts +33 -0
- package/esm/sqlite/columns.d.ts.map +1 -0
- package/esm/sqlite/columns.js +51 -0
- package/esm/sqlite/store.d.ts +73 -25
- package/esm/sqlite/store.d.ts.map +1 -1
- package/esm/sqlite/store.js +411 -107
- package/esm/types.d.ts +1 -1
- package/esm/url.d.ts +122 -19
- package/esm/url.d.ts.map +1 -1
- package/esm/url.js +218 -24
- package/esm/walk-via-ls.d.ts +122 -0
- package/esm/walk-via-ls.d.ts.map +1 -0
- package/esm/walk-via-ls.js +178 -0
- package/package.json +1 -1
- package/esm/byte-entity-shim.d.ts +0 -47
- package/esm/byte-entity-shim.d.ts.map +0 -1
- package/esm/byte-entity-shim.js +0 -138
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/0.22.0/mod.d.ts.map +0 -1
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/0.22.0/src/encrypt/mod.d.ts.map +0 -1
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/0.22.0/src/network/policies/path-vector.d.ts.map +0 -1
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/0.22.0/src/rig/connection.d.ts.map +0 -1
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/0.22.0/src/rig/rig.d.ts.map +0 -1
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/0.22.0/src/rig/types.d.ts.map +0 -1
- package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/0.22.0/src/types/types.d.ts.map +0 -1
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/mod.js +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/client-console/client.d.ts +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/client-console/client.js +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/encoding/encoding.d.ts +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/encoding/encoding.js +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/functional-client/functional-client.d.ts +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/functional-client/functional-client.js +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/match-pattern/match-pattern.d.ts +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/match-pattern/match-pattern.js +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/network/decorators.d.ts +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/network/decorators.js +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/network/network.d.ts +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/network/network.js +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/network/peer.d.ts +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/network/peer.js +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/network/policies/flood.d.ts +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/network/policies/flood.js +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/network/policies/tell-and-read.d.ts +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/network/policies/tell-and-read.js +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/network/types.d.ts +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/network/types.js +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/observe-emitter/observe-emitter.d.ts +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/observe-emitter/observe-emitter.js +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/rig/events.d.ts +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/rig/events.js +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/rig/hooks.d.ts +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/rig/hooks.js +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/rig/identity.d.ts +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/rig/identity.js +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/rig/operation-handle.d.ts +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/rig/operation-handle.js +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/rig/reactions.d.ts +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/rig/reactions.js +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/rig/types.js +0 -0
- /package/esm/deps/jsr.io/@bandeira-tech/b3nd-core/{0.22.0 → 0.25.0}/src/types/types.js +0 -0
package/esm/s3/store.d.ts
CHANGED
|
@@ -1,46 +1,148 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* S3Store — Amazon S3 implementation of
|
|
2
|
+
* S3Store — Amazon S3 implementation of `EntityStore`.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
* object per entry. Object body is the payload bytes verbatim; the
|
|
6
|
-
* store is opaque.
|
|
4
|
+
* One backend, many entities. Layouts:
|
|
7
5
|
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
6
|
+
* - `BYTES_ENTITY` → object key `{prefix}{uriToKey(uri)}.bin` holding
|
|
7
|
+
* the raw payload bytes (existing layout — no migration).
|
|
8
|
+
* - any other schema → object key
|
|
9
|
+
* `{prefix}entities/{entityName}/{uriToKey(uri)}.bin` holding a
|
|
10
|
+
* JSON-encoded record. Per-field canonical `TYPE_TAGS` round-trip
|
|
11
|
+
* the JSON boundary (bytes → base64, bigint → string, timestamp →
|
|
12
|
+
* ISO-8601 — see `./fields.ts`).
|
|
13
|
+
*
|
|
14
|
+
* ## Lifecycle
|
|
15
|
+
*
|
|
16
|
+
* `entitySupport(schema)` is pure — it plans the field set and computes
|
|
17
|
+
* a collision signature. `provisionEntity(meta)` writes a tiny meta
|
|
18
|
+
* object at `{prefix}__meta__/entities/{name}` holding the signature;
|
|
19
|
+
* subsequent `entityStatus(meta)` reads it back.
|
|
20
|
+
*
|
|
21
|
+
* ## Read params
|
|
22
|
+
*
|
|
23
|
+
* `fn=ls` / `fn=count` are shallow direct-leaves only: `listObjects`
|
|
24
|
+
* under the entity's key range, filter to direct-child `.bin` keys
|
|
25
|
+
* (no further `/`). Standard params (`limit`/`page`/`sortBy=uri`/
|
|
26
|
+
* `sortOrder`/`format`) are applied in memory; `format=uris` and
|
|
27
|
+
* `count` skip the per-leaf `getObject`.
|
|
28
|
+
*
|
|
29
|
+
* `fn=find` is the recursive variant (v2 spec §3.3, §3.5): S3's
|
|
30
|
+
* `listObjects(prefix)` already walks every key under the prefix
|
|
31
|
+
* (the flat keyspace is what S3 returns natively — the shallow
|
|
32
|
+
* `tail.includes("/")` drop in `_listChildUris` is an in-process
|
|
33
|
+
* filter we omit for find). Dispatch applies the glob post-filter,
|
|
34
|
+
* sort, cursor, and pagination on top. We declare `pushDownFind:
|
|
35
|
+
* false` honestly: the *prefix* is pushed down (S3 supports nothing
|
|
36
|
+
* else), but the §3.3 glob grammar (`?`, `*`, `**`, mid-`**`) is
|
|
37
|
+
* richer than what S3's list API expresses, so the glob match still
|
|
38
|
+
* happens in dispatch.
|
|
39
|
+
*
|
|
40
|
+
* ### Known limit: listObjects pagination
|
|
41
|
+
*
|
|
42
|
+
* S3's `ListObjectsV2` returns at most 1000 keys per response and
|
|
43
|
+
* paginates via `ContinuationToken`. The current `S3Executor.listObjects`
|
|
44
|
+
* contract returns a flat `string[]` — executors are expected to walk
|
|
45
|
+
* the continuation themselves and concatenate. The in-tree integration
|
|
46
|
+
* executor (`./integration.test.ts`) does NOT yet paginate, so any
|
|
47
|
+
* entity holding >1000 records will silently truncate in the
|
|
48
|
+
* integration suite. Production executors built on the AWS SDK
|
|
49
|
+
* (e.g. via `@aws-sdk/client-s3`'s paginator) get this for free.
|
|
50
|
+
*
|
|
51
|
+
* ### Push-down matrix
|
|
52
|
+
*
|
|
53
|
+
* - `pushDownPattern`: not advertised — dispatch runs the glob.
|
|
54
|
+
* - `pushDownCursor`: not advertised — dispatch slices in JS.
|
|
55
|
+
* - `pushDownSortBy`: not advertised — dispatch sorts in JS.
|
|
56
|
+
* - `pushDownFind`: false (see above).
|
|
57
|
+
* - `pushDownCount`: not advertised — S3 has no efficient COUNT
|
|
58
|
+
* (a COUNT translation would require streaming every key under
|
|
59
|
+
* the prefix and tallying). For deep counts (`**`), dispatch
|
|
60
|
+
* walks via the find handler and returns the length.
|
|
13
61
|
*/
|
|
14
|
-
import type { DeleteResult, Output, StatusResult } from "../deps/jsr.io/@bandeira-tech/b3nd-core/0.
|
|
15
|
-
import { type BytesOnlyEntityMeta } from "../byte-entity-shim.js";
|
|
62
|
+
import type { DeleteResult, Output, StatusResult } from "../deps/jsr.io/@bandeira-tech/b3nd-core/0.25.0/src/types/types.js";
|
|
16
63
|
import type { EntityStore } from "../entity-store.js";
|
|
17
|
-
import type
|
|
64
|
+
import { type EntityMeta, type EntityRecord, type EntitySchema } from "../entity.js";
|
|
18
65
|
import type { StoreCapabilities, StoreWriteResult } from "../types.js";
|
|
66
|
+
import { type FieldPlan } from "../json-fields.js";
|
|
19
67
|
import type { S3Executor } from "./mod.js";
|
|
20
|
-
|
|
68
|
+
/**
|
|
69
|
+
* S3Store-specific entity handle.
|
|
70
|
+
*
|
|
71
|
+
* `keyPrefix` is the S3 key root for this entity's objects (the
|
|
72
|
+
* store's `prefix` for BYTES_ENTITY, plus `entities/{name}/` for
|
|
73
|
+
* custom). `isBytes` selects the raw-payload object body vs the
|
|
74
|
+
* JSON-encoded record body. `fields`/`declared`/`bytesFields` drive
|
|
75
|
+
* write-time validation and encoding; `signature` is used for
|
|
76
|
+
* collision detection.
|
|
77
|
+
*/
|
|
78
|
+
export interface S3EntityMeta extends EntityMeta {
|
|
79
|
+
readonly keyPrefix: string;
|
|
80
|
+
readonly isBytes: boolean;
|
|
81
|
+
readonly fields: readonly FieldPlan[];
|
|
82
|
+
readonly declared: ReadonlySet<string>;
|
|
83
|
+
readonly bytesFields: ReadonlySet<string>;
|
|
84
|
+
readonly signature: string;
|
|
85
|
+
}
|
|
86
|
+
export declare class S3Store implements EntityStore<S3EntityMeta> {
|
|
21
87
|
private readonly bucket;
|
|
22
88
|
private readonly executor;
|
|
23
89
|
private readonly prefix;
|
|
24
90
|
constructor(bucket: string, executor: S3Executor, prefix?: string);
|
|
25
|
-
entitySupport(schema: EntitySchema):
|
|
26
|
-
entityStatus(meta:
|
|
27
|
-
provisionEntity(
|
|
28
|
-
write(meta:
|
|
91
|
+
entitySupport(schema: EntitySchema): S3EntityMeta;
|
|
92
|
+
entityStatus(meta: S3EntityMeta): Promise<"live" | "unprovisioned">;
|
|
93
|
+
provisionEntity(meta: S3EntityMeta): Promise<void>;
|
|
94
|
+
write(meta: S3EntityMeta, entries: {
|
|
29
95
|
uri: string;
|
|
30
96
|
record: EntityRecord;
|
|
31
97
|
}[]): Promise<StoreWriteResult[]>;
|
|
32
|
-
read<T = EntityRecord | undefined>(meta:
|
|
33
|
-
delete(meta: BytesOnlyEntityMeta, uris: string[]): Promise<DeleteResult[]>;
|
|
34
|
-
private _writeBytes;
|
|
35
|
-
private _readBytes;
|
|
98
|
+
read<T = EntityRecord | undefined>(meta: S3EntityMeta, urls: string[]): Promise<Output<T>[]>;
|
|
36
99
|
private _readOne;
|
|
37
100
|
/** List direct-child URIs under a prefix (no recursion into sub-prefixes). */
|
|
38
101
|
private _listChildUris;
|
|
102
|
+
/**
|
|
103
|
+
* List every descendant URI under a prefix (deep walk). Powers
|
|
104
|
+
* `fn=find`. Unlike `_listChildUris`, this keeps keys whose tail
|
|
105
|
+
* contains `/` — S3's `listObjects(prefix)` already returns every
|
|
106
|
+
* key under the prefix (flat keyspace), so the recursive walk is
|
|
107
|
+
* "the same call without the shallow drop". Dispatch's
|
|
108
|
+
* `compileSaveGlob` post-filter then applies the user's deep glob
|
|
109
|
+
* pattern (e.g. `**` or `**` + `/sub/*.md`), sort, cursor, and
|
|
110
|
+
* pagination.
|
|
111
|
+
*
|
|
112
|
+
* Pagination: relies on `S3Executor.listObjects` to walk
|
|
113
|
+
* continuation tokens itself (see the module-level comment about
|
|
114
|
+
* the in-tree integration executor's known truncation).
|
|
115
|
+
*/
|
|
116
|
+
private _listAllDescendantUris;
|
|
39
117
|
private _ls;
|
|
40
118
|
private _count;
|
|
41
|
-
|
|
119
|
+
/**
|
|
120
|
+
* `fn=find` handler: deep walk under the prefix, return Output[] or
|
|
121
|
+
* string[] depending on `format`. dispatch runs the glob post-filter,
|
|
122
|
+
* sort, pagination, and cursor slot on top.
|
|
123
|
+
*/
|
|
124
|
+
private _find;
|
|
125
|
+
delete(meta: S3EntityMeta, uris: string[]): Promise<DeleteResult[]>;
|
|
42
126
|
status(): Promise<StatusResult>;
|
|
43
127
|
capabilities(): StoreCapabilities;
|
|
44
|
-
|
|
128
|
+
/** Full S3 key for a record under an entity. */
|
|
129
|
+
private _dataKey;
|
|
130
|
+
/** Full S3 key for an entity's meta object. */
|
|
131
|
+
private _metaKey;
|
|
132
|
+
/**
|
|
133
|
+
* Read the persisted signature for an entity, or `null` if no
|
|
134
|
+
* provisioning record exists.
|
|
135
|
+
*/
|
|
136
|
+
private _readMetaSignature;
|
|
137
|
+
/**
|
|
138
|
+
* Scan the meta keyspace for every provisioned entity name. Used by
|
|
139
|
+
* `status()` to advertise the entities a peer can talk to.
|
|
140
|
+
*/
|
|
141
|
+
private _listProvisionedEntities;
|
|
142
|
+
/**
|
|
143
|
+
* Collect any `ReadableStream` values on `bytes`-tagged fields into
|
|
144
|
+
* `Uint8Array` and return a shallow-copied record.
|
|
145
|
+
*/
|
|
146
|
+
private _normaliseBytesFields;
|
|
45
147
|
}
|
|
46
148
|
//# sourceMappingURL=store.d.ts.map
|
package/esm/s3/store.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/s3/store.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/s3/store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4DG;AAEH,OAAO,KAAK,EACV,YAAY,EACZ,MAAM,EACN,YAAY,EACb,MAAM,mEAAmE,CAAC;AAM3E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,YAAY,EACjB,KAAK,YAAY,EAClB,MAAM,cAAc,CAAC;AAEtB,OAAO,KAAK,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACvE,OAAO,EAIL,KAAK,SAAS,EAEf,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAmB3C;;;;;;;;;GASG;AACH,MAAM,WAAW,YAAa,SAAQ,UAAU;IAC9C,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,SAAS,SAAS,EAAE,CAAC;IACtC,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACvC,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAC1C,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAMD,qBAAa,OAAQ,YAAW,WAAW,CAAC,YAAY,CAAC;IACvD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAa;IACtC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;gBAEpB,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,CAAC,EAAE,MAAM;IAWjE,aAAa,CAAC,MAAM,EAAE,YAAY,GAAG,YAAY;IA0C3C,YAAY,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,GAAG,eAAe,CAAC;IAMnE,eAAe,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBlD,KAAK,CACT,IAAI,EAAE,YAAY,EAClB,OAAO,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,YAAY,CAAA;KAAE,EAAE,GAC/C,OAAO,CAAC,gBAAgB,EAAE,CAAC;IA2E9B,IAAI,CAAC,CAAC,GAAG,YAAY,GAAG,SAAS,EAC/B,IAAI,EAAE,YAAY,EAClB,IAAI,EAAE,MAAM,EAAE,GACb,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;YAcT,QAAQ;IAgBtB,8EAA8E;YAChE,cAAc;IAgB5B;;;;;;;;;;;;;OAaG;YACW,sBAAsB;YAetB,GAAG;YA4BH,MAAM;IAYpB;;;;OAIG;YACW,KAAK;IAoBb,MAAM,CAAC,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAiCnE,MAAM,IAAI,OAAO,CAAC,YAAY,CAAC;IA2BrC,YAAY,IAAI,iBAAiB;IAMjC,gDAAgD;IAChD,OAAO,CAAC,QAAQ;IAIhB,+CAA+C;IAC/C,OAAO,CAAC,QAAQ;IAIhB;;;OAGG;YACW,kBAAkB;IAahC;;;OAGG;YACW,wBAAwB;IAYtC;;;OAGG;YACW,qBAAqB;CAmBpC"}
|
package/esm/s3/store.js
CHANGED
|
@@ -1,22 +1,75 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* S3Store — Amazon S3 implementation of
|
|
2
|
+
* S3Store — Amazon S3 implementation of `EntityStore`.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
* object per entry. Object body is the payload bytes verbatim; the
|
|
6
|
-
* store is opaque.
|
|
4
|
+
* One backend, many entities. Layouts:
|
|
7
5
|
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
6
|
+
* - `BYTES_ENTITY` → object key `{prefix}{uriToKey(uri)}.bin` holding
|
|
7
|
+
* the raw payload bytes (existing layout — no migration).
|
|
8
|
+
* - any other schema → object key
|
|
9
|
+
* `{prefix}entities/{entityName}/{uriToKey(uri)}.bin` holding a
|
|
10
|
+
* JSON-encoded record. Per-field canonical `TYPE_TAGS` round-trip
|
|
11
|
+
* the JSON boundary (bytes → base64, bigint → string, timestamp →
|
|
12
|
+
* ISO-8601 — see `./fields.ts`).
|
|
13
|
+
*
|
|
14
|
+
* ## Lifecycle
|
|
15
|
+
*
|
|
16
|
+
* `entitySupport(schema)` is pure — it plans the field set and computes
|
|
17
|
+
* a collision signature. `provisionEntity(meta)` writes a tiny meta
|
|
18
|
+
* object at `{prefix}__meta__/entities/{name}` holding the signature;
|
|
19
|
+
* subsequent `entityStatus(meta)` reads it back.
|
|
20
|
+
*
|
|
21
|
+
* ## Read params
|
|
22
|
+
*
|
|
23
|
+
* `fn=ls` / `fn=count` are shallow direct-leaves only: `listObjects`
|
|
24
|
+
* under the entity's key range, filter to direct-child `.bin` keys
|
|
25
|
+
* (no further `/`). Standard params (`limit`/`page`/`sortBy=uri`/
|
|
26
|
+
* `sortOrder`/`format`) are applied in memory; `format=uris` and
|
|
27
|
+
* `count` skip the per-leaf `getObject`.
|
|
28
|
+
*
|
|
29
|
+
* `fn=find` is the recursive variant (v2 spec §3.3, §3.5): S3's
|
|
30
|
+
* `listObjects(prefix)` already walks every key under the prefix
|
|
31
|
+
* (the flat keyspace is what S3 returns natively — the shallow
|
|
32
|
+
* `tail.includes("/")` drop in `_listChildUris` is an in-process
|
|
33
|
+
* filter we omit for find). Dispatch applies the glob post-filter,
|
|
34
|
+
* sort, cursor, and pagination on top. We declare `pushDownFind:
|
|
35
|
+
* false` honestly: the *prefix* is pushed down (S3 supports nothing
|
|
36
|
+
* else), but the §3.3 glob grammar (`?`, `*`, `**`, mid-`**`) is
|
|
37
|
+
* richer than what S3's list API expresses, so the glob match still
|
|
38
|
+
* happens in dispatch.
|
|
39
|
+
*
|
|
40
|
+
* ### Known limit: listObjects pagination
|
|
41
|
+
*
|
|
42
|
+
* S3's `ListObjectsV2` returns at most 1000 keys per response and
|
|
43
|
+
* paginates via `ContinuationToken`. The current `S3Executor.listObjects`
|
|
44
|
+
* contract returns a flat `string[]` — executors are expected to walk
|
|
45
|
+
* the continuation themselves and concatenate. The in-tree integration
|
|
46
|
+
* executor (`./integration.test.ts`) does NOT yet paginate, so any
|
|
47
|
+
* entity holding >1000 records will silently truncate in the
|
|
48
|
+
* integration suite. Production executors built on the AWS SDK
|
|
49
|
+
* (e.g. via `@aws-sdk/client-s3`'s paginator) get this for free.
|
|
50
|
+
*
|
|
51
|
+
* ### Push-down matrix
|
|
52
|
+
*
|
|
53
|
+
* - `pushDownPattern`: not advertised — dispatch runs the glob.
|
|
54
|
+
* - `pushDownCursor`: not advertised — dispatch slices in JS.
|
|
55
|
+
* - `pushDownSortBy`: not advertised — dispatch sorts in JS.
|
|
56
|
+
* - `pushDownFind`: false (see above).
|
|
57
|
+
* - `pushDownCount`: not advertised — S3 has no efficient COUNT
|
|
58
|
+
* (a COUNT translation would require streaming every key under
|
|
59
|
+
* the prefix and tallying). For deep counts (`**`), dispatch
|
|
60
|
+
* walks via the find handler and returns the length.
|
|
13
61
|
*/
|
|
14
|
-
import { bytesOnlyDelete, bytesOnlyEntitySupport, bytesOnlyRead, bytesOnlyWrite, } from "../byte-entity-shim.js";
|
|
15
62
|
import { dispatchRead } from "../dispatch.js";
|
|
16
63
|
import { storageFailure } from "../errors.js";
|
|
64
|
+
import { toBytes } from "../payload.js";
|
|
17
65
|
import { validateReadParams } from "../read.js";
|
|
66
|
+
import { BYTES_ENTITY, } from "../entity.js";
|
|
67
|
+
import { IDENTIFIER_PATTERN, isValidIdentifier } from "../identifiers.js";
|
|
68
|
+
import { computeSignature, decodeRecord, encodeRecord, planFields, } from "../json-fields.js";
|
|
18
69
|
const STORE_NAME = "S3Store";
|
|
19
70
|
const EXT = ".bin";
|
|
71
|
+
const ENTITY_KEY_PREFIX = "entities";
|
|
72
|
+
const META_KEY_PREFIX = "__meta__/entities";
|
|
20
73
|
/** `protocol://host/path` → `protocol_host/path`. */
|
|
21
74
|
function uriToKey(uri) {
|
|
22
75
|
return uri.replace("://", "_").replace(/^\//, "");
|
|
@@ -26,6 +79,9 @@ function keyTailToUri(tail) {
|
|
|
26
79
|
const noExt = tail.endsWith(EXT) ? tail.slice(0, -EXT.length) : tail;
|
|
27
80
|
return noExt.replace("_", "://");
|
|
28
81
|
}
|
|
82
|
+
function isBytesSchema(schema) {
|
|
83
|
+
return schema.name === BYTES_ENTITY.name;
|
|
84
|
+
}
|
|
29
85
|
export class S3Store {
|
|
30
86
|
bucket;
|
|
31
87
|
executor;
|
|
@@ -39,56 +95,138 @@ export class S3Store {
|
|
|
39
95
|
this.executor = executor;
|
|
40
96
|
this.prefix = prefix ?? "";
|
|
41
97
|
}
|
|
42
|
-
// ──
|
|
98
|
+
// ── Lifecycle ────────────────────────────────────────────────────
|
|
43
99
|
entitySupport(schema) {
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
100
|
+
if (isBytesSchema(schema)) {
|
|
101
|
+
const fields = [{ name: "payload", tag: "bytes" }];
|
|
102
|
+
return {
|
|
103
|
+
support: {
|
|
104
|
+
entity: schema.name,
|
|
105
|
+
supported: ["payload"],
|
|
106
|
+
unsupported: [],
|
|
107
|
+
},
|
|
108
|
+
keyPrefix: this.prefix,
|
|
109
|
+
isBytes: true,
|
|
110
|
+
fields,
|
|
111
|
+
declared: new Set(["payload"]),
|
|
112
|
+
bytesFields: new Set(["payload"]),
|
|
113
|
+
signature: computeSignature(schema.name, fields),
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
if (!isValidIdentifier(schema.name)) {
|
|
117
|
+
throw new Error(`${STORE_NAME}: entity name '${schema.name}' must match ${IDENTIFIER_PATTERN.source}`);
|
|
118
|
+
}
|
|
119
|
+
const { fields, unsupported } = planFields(schema.fields);
|
|
120
|
+
const declared = new Set(fields.map((f) => f.name));
|
|
121
|
+
const bytesFields = new Set(fields.filter((f) => f.tag === "bytes").map((f) => f.name));
|
|
122
|
+
return {
|
|
123
|
+
support: {
|
|
124
|
+
entity: schema.name,
|
|
125
|
+
supported: fields.map((f) => f.name),
|
|
126
|
+
unsupported,
|
|
127
|
+
},
|
|
128
|
+
keyPrefix: `${this.prefix}${ENTITY_KEY_PREFIX}/${schema.name}/`,
|
|
129
|
+
isBytes: false,
|
|
130
|
+
fields,
|
|
131
|
+
declared,
|
|
132
|
+
bytesFields,
|
|
133
|
+
signature: computeSignature(schema.name, fields),
|
|
134
|
+
};
|
|
54
135
|
}
|
|
55
|
-
|
|
56
|
-
|
|
136
|
+
async entityStatus(meta) {
|
|
137
|
+
const recorded = await this._readMetaSignature(meta.support.entity);
|
|
138
|
+
if (recorded === null)
|
|
139
|
+
return "unprovisioned";
|
|
140
|
+
return recorded === meta.signature ? "live" : "unprovisioned";
|
|
57
141
|
}
|
|
58
|
-
|
|
59
|
-
|
|
142
|
+
async provisionEntity(meta) {
|
|
143
|
+
const recorded = await this._readMetaSignature(meta.support.entity);
|
|
144
|
+
if (recorded !== null) {
|
|
145
|
+
if (recorded !== meta.signature) {
|
|
146
|
+
throw new Error(`${STORE_NAME}: entity '${meta.support.entity}' is already ` +
|
|
147
|
+
`provisioned with a different shape`);
|
|
148
|
+
}
|
|
149
|
+
return;
|
|
150
|
+
}
|
|
151
|
+
await this.executor.putObject(this._metaKey(meta.support.entity), new TextEncoder().encode(meta.signature), "text/plain");
|
|
60
152
|
}
|
|
61
|
-
// ──
|
|
62
|
-
async
|
|
153
|
+
// ── Write ────────────────────────────────────────────────────────
|
|
154
|
+
async write(meta, entries) {
|
|
155
|
+
if (entries.length === 0)
|
|
156
|
+
return [];
|
|
157
|
+
if ((await this._readMetaSignature(meta.support.entity)) !== meta.signature) {
|
|
158
|
+
return entries.map(({ uri }) => ({
|
|
159
|
+
success: false,
|
|
160
|
+
...storageFailure(new Error(`${STORE_NAME}: entity '${meta.support.entity}' is not provisioned`), "Entity not provisioned", uri),
|
|
161
|
+
}));
|
|
162
|
+
}
|
|
63
163
|
const results = [];
|
|
64
|
-
for (const
|
|
164
|
+
for (const { uri, record } of entries) {
|
|
165
|
+
const extras = Object.keys(record).filter((k) => !meta.declared.has(k));
|
|
166
|
+
if (extras.length > 0) {
|
|
167
|
+
results.push({
|
|
168
|
+
success: false,
|
|
169
|
+
...storageFailure(new Error(`record contains keys not declared in schema '${meta.support.entity}': ${extras.join(", ")}`), "Schema mismatch", uri),
|
|
170
|
+
});
|
|
171
|
+
continue;
|
|
172
|
+
}
|
|
65
173
|
try {
|
|
66
|
-
|
|
174
|
+
if (meta.isBytes) {
|
|
175
|
+
const payload = record.payload;
|
|
176
|
+
if (!(payload instanceof Uint8Array) &&
|
|
177
|
+
!(payload instanceof ReadableStream)) {
|
|
178
|
+
throw new Error("BYTES_ENTITY record.payload must be Uint8Array or ReadableStream");
|
|
179
|
+
}
|
|
180
|
+
await this.executor.putObject(this._dataKey(meta, uri), payload, "application/octet-stream");
|
|
181
|
+
}
|
|
182
|
+
else {
|
|
183
|
+
const normalised = await this._normaliseBytesFields(record, meta.bytesFields);
|
|
184
|
+
const json = JSON.stringify(encodeRecord(meta.fields, normalised));
|
|
185
|
+
await this.executor.putObject(this._dataKey(meta, uri), new TextEncoder().encode(json), "application/json");
|
|
186
|
+
}
|
|
67
187
|
results.push({ success: true });
|
|
68
188
|
}
|
|
69
189
|
catch (err) {
|
|
70
190
|
results.push({
|
|
71
191
|
success: false,
|
|
72
|
-
...storageFailure(err, "Write failed",
|
|
192
|
+
...storageFailure(err, "Write failed", uri),
|
|
73
193
|
});
|
|
74
194
|
}
|
|
75
195
|
}
|
|
76
196
|
return results;
|
|
77
197
|
}
|
|
78
|
-
|
|
198
|
+
// ── Read ─────────────────────────────────────────────────────────
|
|
199
|
+
read(meta, urls) {
|
|
79
200
|
return dispatchRead(urls, STORE_NAME, {
|
|
80
|
-
read: (p) => this._readOne(p.uri),
|
|
81
|
-
ls: (p) => this._ls(p),
|
|
82
|
-
count: (p) => this._count(p),
|
|
201
|
+
read: (p) => this._readOne(meta, p.uri),
|
|
202
|
+
ls: (p) => this._ls(meta, p),
|
|
203
|
+
count: (p) => this._count(meta, p),
|
|
204
|
+
find: (p) => this._find(meta, p),
|
|
205
|
+
// The handler returns every descendant URI under the prefix;
|
|
206
|
+
// dispatch's `compileSaveGlob` post-filter applies the §3.3
|
|
207
|
+
// grammar (`?`, `*`, `**`, mid-`**`). S3's list API only
|
|
208
|
+
// expresses prefix matching, so push-down stops at the prefix.
|
|
209
|
+
pushDownFind: false,
|
|
83
210
|
});
|
|
84
211
|
}
|
|
85
|
-
async _readOne(uri) {
|
|
86
|
-
const
|
|
87
|
-
|
|
212
|
+
async _readOne(meta, uri) {
|
|
213
|
+
const stream = await this.executor.getObject(this._dataKey(meta, uri));
|
|
214
|
+
if (!stream)
|
|
215
|
+
return undefined;
|
|
216
|
+
if (meta.isBytes)
|
|
217
|
+
return { payload: stream };
|
|
218
|
+
try {
|
|
219
|
+
const bytes = await new Response(stream).bytes();
|
|
220
|
+
const text = new TextDecoder().decode(bytes);
|
|
221
|
+
return decodeRecord(meta.fields, JSON.parse(text));
|
|
222
|
+
}
|
|
223
|
+
catch {
|
|
224
|
+
return undefined;
|
|
225
|
+
}
|
|
88
226
|
}
|
|
89
227
|
/** List direct-child URIs under a prefix (no recursion into sub-prefixes). */
|
|
90
|
-
async _listChildUris(prefixUri) {
|
|
91
|
-
const keyPrefix = `${
|
|
228
|
+
async _listChildUris(meta, prefixUri) {
|
|
229
|
+
const keyPrefix = `${meta.keyPrefix}${uriToKey(prefixUri)}`;
|
|
92
230
|
const keys = await this.executor.listObjects(keyPrefix);
|
|
93
231
|
const uris = [];
|
|
94
232
|
for (const key of keys) {
|
|
@@ -96,16 +234,42 @@ export class S3Store {
|
|
|
96
234
|
continue;
|
|
97
235
|
const tail = key.slice(keyPrefix.length);
|
|
98
236
|
if (tail.includes("/"))
|
|
99
|
-
continue;
|
|
237
|
+
continue;
|
|
238
|
+
uris.push(`${prefixUri}${keyTailToUri(tail)}`);
|
|
239
|
+
}
|
|
240
|
+
return uris;
|
|
241
|
+
}
|
|
242
|
+
/**
|
|
243
|
+
* List every descendant URI under a prefix (deep walk). Powers
|
|
244
|
+
* `fn=find`. Unlike `_listChildUris`, this keeps keys whose tail
|
|
245
|
+
* contains `/` — S3's `listObjects(prefix)` already returns every
|
|
246
|
+
* key under the prefix (flat keyspace), so the recursive walk is
|
|
247
|
+
* "the same call without the shallow drop". Dispatch's
|
|
248
|
+
* `compileSaveGlob` post-filter then applies the user's deep glob
|
|
249
|
+
* pattern (e.g. `**` or `**` + `/sub/*.md`), sort, cursor, and
|
|
250
|
+
* pagination.
|
|
251
|
+
*
|
|
252
|
+
* Pagination: relies on `S3Executor.listObjects` to walk
|
|
253
|
+
* continuation tokens itself (see the module-level comment about
|
|
254
|
+
* the in-tree integration executor's known truncation).
|
|
255
|
+
*/
|
|
256
|
+
async _listAllDescendantUris(meta, prefixUri) {
|
|
257
|
+
const keyPrefix = `${meta.keyPrefix}${uriToKey(prefixUri)}`;
|
|
258
|
+
const keys = await this.executor.listObjects(keyPrefix);
|
|
259
|
+
const uris = [];
|
|
260
|
+
for (const key of keys) {
|
|
261
|
+
if (!key.endsWith(EXT))
|
|
262
|
+
continue;
|
|
263
|
+
const tail = key.slice(keyPrefix.length);
|
|
100
264
|
uris.push(`${prefixUri}${keyTailToUri(tail)}`);
|
|
101
265
|
}
|
|
102
266
|
return uris;
|
|
103
267
|
}
|
|
104
|
-
async _ls(parsed) {
|
|
268
|
+
async _ls(meta, parsed) {
|
|
105
269
|
validateReadParams(parsed.params, STORE_NAME);
|
|
106
270
|
const { params } = parsed;
|
|
107
271
|
const format = params.format ?? "full";
|
|
108
|
-
let uris = await this._listChildUris(parsed.uri);
|
|
272
|
+
let uris = await this._listChildUris(meta, parsed.uri);
|
|
109
273
|
if (params.sortBy === "uri") {
|
|
110
274
|
const dir = params.sortOrder === "desc" ? -1 : 1;
|
|
111
275
|
uris = [...uris].sort((a, b) => a.localeCompare(b) * dir);
|
|
@@ -119,21 +283,56 @@ export class S3Store {
|
|
|
119
283
|
return uris;
|
|
120
284
|
const out = [];
|
|
121
285
|
for (const uri of uris) {
|
|
122
|
-
|
|
286
|
+
const rec = await this._readOne(meta, uri);
|
|
287
|
+
if (rec !== undefined)
|
|
288
|
+
out.push([uri, rec]);
|
|
123
289
|
}
|
|
124
290
|
return out;
|
|
125
291
|
}
|
|
126
|
-
async _count(parsed) {
|
|
292
|
+
async _count(meta, parsed) {
|
|
127
293
|
if (parsed.params.pattern !== undefined) {
|
|
294
|
+
// Filtered count routes through dispatch's ls-and-count path,
|
|
295
|
+
// which strips `pattern` before calling our `ls` handler — we
|
|
296
|
+
// should never see `params.pattern` here. Throw loudly if we do:
|
|
297
|
+
// it means dispatch's contract changed and the post-filter is
|
|
298
|
+
// about to silently undercount.
|
|
128
299
|
throw new Error(`${STORE_NAME}: pattern filter not supported`);
|
|
129
300
|
}
|
|
130
|
-
return (await this._listChildUris(parsed.uri)).length;
|
|
301
|
+
return (await this._listChildUris(meta, parsed.uri)).length;
|
|
131
302
|
}
|
|
132
|
-
|
|
303
|
+
/**
|
|
304
|
+
* `fn=find` handler: deep walk under the prefix, return Output[] or
|
|
305
|
+
* string[] depending on `format`. dispatch runs the glob post-filter,
|
|
306
|
+
* sort, pagination, and cursor slot on top.
|
|
307
|
+
*/
|
|
308
|
+
async _find(meta, parsed) {
|
|
309
|
+
validateReadParams(parsed.params, STORE_NAME);
|
|
310
|
+
const format = parsed.params.format ?? "full";
|
|
311
|
+
const uris = await this._listAllDescendantUris(meta, parsed.uri);
|
|
312
|
+
if (format === "uris")
|
|
313
|
+
return uris;
|
|
314
|
+
const out = [];
|
|
315
|
+
for (const uri of uris) {
|
|
316
|
+
const rec = await this._readOne(meta, uri);
|
|
317
|
+
if (rec !== undefined)
|
|
318
|
+
out.push([uri, rec]);
|
|
319
|
+
}
|
|
320
|
+
return out;
|
|
321
|
+
}
|
|
322
|
+
// ── Delete ───────────────────────────────────────────────────────
|
|
323
|
+
async delete(meta, uris) {
|
|
324
|
+
if (uris.length === 0)
|
|
325
|
+
return [];
|
|
326
|
+
if ((await this._readMetaSignature(meta.support.entity)) !== meta.signature) {
|
|
327
|
+
return uris.map((uri) => ({
|
|
328
|
+
success: false,
|
|
329
|
+
...storageFailure(new Error(`${STORE_NAME}: entity '${meta.support.entity}' is not provisioned`), "Entity not provisioned", uri),
|
|
330
|
+
}));
|
|
331
|
+
}
|
|
133
332
|
const results = [];
|
|
134
333
|
for (const uri of uris) {
|
|
135
334
|
try {
|
|
136
|
-
await this.executor.deleteObject(this.
|
|
335
|
+
await this.executor.deleteObject(this._dataKey(meta, uri));
|
|
137
336
|
results.push({ success: true });
|
|
138
337
|
}
|
|
139
338
|
catch (err) {
|
|
@@ -145,7 +344,7 @@ export class S3Store {
|
|
|
145
344
|
}
|
|
146
345
|
return results;
|
|
147
346
|
}
|
|
148
|
-
// ── Status
|
|
347
|
+
// ── Status / capabilities ────────────────────────────────────────
|
|
149
348
|
async status() {
|
|
150
349
|
try {
|
|
151
350
|
const ok = await this.executor.headBucket();
|
|
@@ -153,31 +352,88 @@ export class S3Store {
|
|
|
153
352
|
return {
|
|
154
353
|
status: "unhealthy",
|
|
155
354
|
message: `Bucket not accessible: ${this.bucket}`,
|
|
156
|
-
fns: ["read", "ls", "count"],
|
|
355
|
+
fns: ["read", "ls", "find", "count"],
|
|
157
356
|
};
|
|
158
357
|
}
|
|
358
|
+
const schema = await this._listProvisionedEntities();
|
|
159
359
|
return {
|
|
160
360
|
status: "healthy",
|
|
161
361
|
message: "S3 store is operational",
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
prefix: this.prefix || "(none)",
|
|
166
|
-
},
|
|
362
|
+
schema: schema.map((s) => `entity:${s}`),
|
|
363
|
+
fns: ["read", "ls", "find", "count"],
|
|
364
|
+
details: { bucket: this.bucket, prefix: this.prefix || "(none)" },
|
|
167
365
|
};
|
|
168
366
|
}
|
|
169
367
|
catch (error) {
|
|
170
368
|
return {
|
|
171
369
|
status: "unhealthy",
|
|
172
370
|
message: error instanceof Error ? error.message : String(error),
|
|
173
|
-
fns: ["read", "ls", "count"],
|
|
371
|
+
fns: ["read", "ls", "find", "count"],
|
|
174
372
|
};
|
|
175
373
|
}
|
|
176
374
|
}
|
|
177
375
|
capabilities() {
|
|
178
376
|
return { atomicBatch: false };
|
|
179
377
|
}
|
|
180
|
-
|
|
181
|
-
|
|
378
|
+
// ── Internals ────────────────────────────────────────────────────
|
|
379
|
+
/** Full S3 key for a record under an entity. */
|
|
380
|
+
_dataKey(meta, uri) {
|
|
381
|
+
return `${meta.keyPrefix}${uriToKey(uri)}${EXT}`;
|
|
382
|
+
}
|
|
383
|
+
/** Full S3 key for an entity's meta object. */
|
|
384
|
+
_metaKey(entityName) {
|
|
385
|
+
return `${this.prefix}${META_KEY_PREFIX}/${entityName}`;
|
|
386
|
+
}
|
|
387
|
+
/**
|
|
388
|
+
* Read the persisted signature for an entity, or `null` if no
|
|
389
|
+
* provisioning record exists.
|
|
390
|
+
*/
|
|
391
|
+
async _readMetaSignature(entityName) {
|
|
392
|
+
try {
|
|
393
|
+
const stream = await this.executor.getObject(this._metaKey(entityName));
|
|
394
|
+
if (!stream)
|
|
395
|
+
return null;
|
|
396
|
+
const bytes = await new Response(stream).bytes();
|
|
397
|
+
return new TextDecoder().decode(bytes);
|
|
398
|
+
}
|
|
399
|
+
catch {
|
|
400
|
+
return null;
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
/**
|
|
404
|
+
* Scan the meta keyspace for every provisioned entity name. Used by
|
|
405
|
+
* `status()` to advertise the entities a peer can talk to.
|
|
406
|
+
*/
|
|
407
|
+
async _listProvisionedEntities() {
|
|
408
|
+
const metaKeyPrefix = `${this.prefix}${META_KEY_PREFIX}/`;
|
|
409
|
+
try {
|
|
410
|
+
const keys = await this.executor.listObjects(metaKeyPrefix);
|
|
411
|
+
return keys
|
|
412
|
+
.map((k) => k.slice(metaKeyPrefix.length))
|
|
413
|
+
.filter((k) => k.length > 0 && !k.includes("/"));
|
|
414
|
+
}
|
|
415
|
+
catch {
|
|
416
|
+
return [];
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
/**
|
|
420
|
+
* Collect any `ReadableStream` values on `bytes`-tagged fields into
|
|
421
|
+
* `Uint8Array` and return a shallow-copied record.
|
|
422
|
+
*/
|
|
423
|
+
async _normaliseBytesFields(record, bytesFields) {
|
|
424
|
+
const out = { ...record };
|
|
425
|
+
for (const name of bytesFields) {
|
|
426
|
+
const v = out[name];
|
|
427
|
+
if (v === undefined || v === null)
|
|
428
|
+
continue;
|
|
429
|
+
if (v instanceof Uint8Array)
|
|
430
|
+
continue;
|
|
431
|
+
if (v instanceof ReadableStream) {
|
|
432
|
+
out[name] = await toBytes(v);
|
|
433
|
+
continue;
|
|
434
|
+
}
|
|
435
|
+
throw new Error(`field '${name}' must be Uint8Array or ReadableStream, got ${typeof v}`);
|
|
436
|
+
}
|
|
437
|
+
return out;
|
|
182
438
|
}
|
|
183
439
|
}
|