@spine-event-engine/storage-datastore 2.0.0-snapshot.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/LICENSE +201 -0
  2. package/README.md +262 -0
  3. package/REFERENCE.md +147 -0
  4. package/dist/datastore/column-mapping.d.ts +28 -0
  5. package/dist/datastore/column-mapping.d.ts.map +1 -0
  6. package/dist/datastore/column-mapping.js +141 -0
  7. package/dist/datastore/column-mapping.js.map +1 -0
  8. package/dist/datastore/delivery-cleanup.d.ts +31 -0
  9. package/dist/datastore/delivery-cleanup.d.ts.map +1 -0
  10. package/dist/datastore/delivery-cleanup.js +111 -0
  11. package/dist/datastore/delivery-cleanup.js.map +1 -0
  12. package/dist/datastore/entity-history.d.ts +82 -0
  13. package/dist/datastore/entity-history.d.ts.map +1 -0
  14. package/dist/datastore/entity-history.js +541 -0
  15. package/dist/datastore/entity-history.js.map +1 -0
  16. package/dist/datastore/id-column.d.ts +34 -0
  17. package/dist/datastore/id-column.d.ts.map +1 -0
  18. package/dist/datastore/id-column.js +103 -0
  19. package/dist/datastore/id-column.js.map +1 -0
  20. package/dist/datastore/namespace.d.ts +81 -0
  21. package/dist/datastore/namespace.d.ts.map +1 -0
  22. package/dist/datastore/namespace.js +123 -0
  23. package/dist/datastore/namespace.js.map +1 -0
  24. package/dist/datastore/record-storage.d.ts +223 -0
  25. package/dist/datastore/record-storage.d.ts.map +1 -0
  26. package/dist/datastore/record-storage.js +855 -0
  27. package/dist/datastore/record-storage.js.map +1 -0
  28. package/dist/datastore/storage-factory.d.ts +200 -0
  29. package/dist/datastore/storage-factory.d.ts.map +1 -0
  30. package/dist/datastore/storage-factory.js +224 -0
  31. package/dist/datastore/storage-factory.js.map +1 -0
  32. package/dist/datastore/tenant-catalog.d.ts +50 -0
  33. package/dist/datastore/tenant-catalog.d.ts.map +1 -0
  34. package/dist/datastore/tenant-catalog.js +154 -0
  35. package/dist/datastore/tenant-catalog.js.map +1 -0
  36. package/dist/index.d.ts +6 -0
  37. package/dist/index.d.ts.map +1 -0
  38. package/dist/index.js +19 -0
  39. package/dist/index.js.map +1 -0
  40. package/dist/tsconfig.tsbuildinfo +1 -0
  41. package/index.yaml +1 -0
  42. package/package.json +40 -0
package/LICENSE ADDED
@@ -0,0 +1,201 @@
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ http://www.apache.org/licenses/
4
+
5
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6
+
7
+ 1. Definitions.
8
+
9
+ "License" shall mean the terms and conditions for use, reproduction,
10
+ and distribution as defined by Sections 1 through 9 of this document.
11
+
12
+ "Licensor" shall mean the copyright owner or entity authorized by
13
+ the copyright owner that is granting the License.
14
+
15
+ "Legal Entity" shall mean the union of the acting entity and all
16
+ other entities that control, are controlled by, or are under common
17
+ control with that entity. For the purposes of this definition,
18
+ "control" means (i) the power, direct or indirect, to cause the
19
+ direction or management of such entity, whether by contract or
20
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
21
+ outstanding shares, or (iii) beneficial ownership of such entity.
22
+
23
+ "You" (or "Your") shall mean an individual or Legal Entity
24
+ exercising permissions granted by this License.
25
+
26
+ "Source" form shall mean the preferred form for making modifications,
27
+ including but not limited to software source code, documentation
28
+ source, and configuration files.
29
+
30
+ "Object" form shall mean any form resulting from mechanical
31
+ transformation or translation of a Source form, including but
32
+ not limited to compiled object code, generated documentation,
33
+ and conversions to other media types.
34
+
35
+ "Work" shall mean the work of authorship, whether in Source or
36
+ Object form, made available under the License, as indicated by a
37
+ copyright notice that is included in or attached to the work
38
+ (an example is provided in the Appendix below).
39
+
40
+ "Derivative Works" shall mean any work, whether in Source or Object
41
+ form, that is based on (or derived from) the Work and for which the
42
+ editorial revisions, annotations, elaborations, or other modifications
43
+ represent, as a whole, an original work of authorship. For the purposes
44
+ of this License, Derivative Works shall not include works that remain
45
+ separable from, or merely link (or bind by name) to the interfaces of,
46
+ the Work and Derivative Works thereof.
47
+
48
+ "Contribution" shall mean any work of authorship, including
49
+ the original version of the Work and any modifications or additions
50
+ to that Work or Derivative Works thereof, that is intentionally
51
+ submitted to Licensor for inclusion in the Work by the copyright owner
52
+ or by an individual or Legal Entity authorized to submit on behalf of
53
+ the copyright owner. For the purposes of this definition, "submitted"
54
+ means any form of electronic, verbal, or written communication sent
55
+ to the Licensor or its representatives, including but not limited to
56
+ communication on electronic mailing lists, source code control systems,
57
+ and issue tracking systems that are managed by, or on behalf of, the
58
+ Licensor for the purpose of discussing and improving the Work, but
59
+ excluding communication that is conspicuously marked or otherwise
60
+ designated in writing by the copyright owner as "Not a Contribution."
61
+
62
+ "Contributor" shall mean Licensor and any individual or Legal Entity
63
+ on behalf of whom a Contribution has been received by Licensor and
64
+ subsequently incorporated within the Work.
65
+
66
+ 2. Grant of Copyright License. Subject to the terms and conditions of
67
+ this License, each Contributor hereby grants to You a perpetual,
68
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69
+ copyright license to reproduce, prepare Derivative Works of,
70
+ publicly display, publicly perform, sublicense, and distribute the
71
+ Work and such Derivative Works in Source or Object form.
72
+
73
+ 3. Grant of Patent License. Subject to the terms and conditions of
74
+ this License, each Contributor hereby grants to You a perpetual,
75
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76
+ (except as stated in this section) patent license to make, have made,
77
+ use, offer to sell, sell, import, and otherwise transfer the Work,
78
+ where such license applies only to those patent claims licensable
79
+ by such Contributor that are necessarily infringed by their
80
+ Contribution(s) alone or by combination of their Contribution(s)
81
+ with the Work to which such Contribution(s) was submitted. If You
82
+ institute patent litigation against any entity (including a
83
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
84
+ or a Contribution incorporated within the Work constitutes direct
85
+ or contributory patent infringement, then any patent licenses
86
+ granted to You under this License for that Work shall terminate
87
+ as of the date such litigation is filed.
88
+
89
+ 4. Redistribution. You may reproduce and distribute copies of the
90
+ Work or Derivative Works thereof in any medium, with or without
91
+ modifications, and in Source or Object form, provided that You
92
+ meet the following conditions:
93
+
94
+ (a) You must give any other recipients of the Work or
95
+ Derivative Works a copy of this License; and
96
+
97
+ (b) You must cause any modified files to carry prominent notices
98
+ stating that You changed the files; and
99
+
100
+ (c) You must retain, in the Source form of any Derivative Works
101
+ that You distribute, all copyright, patent, trademark, and
102
+ attribution notices from the Source form of the Work,
103
+ excluding those notices that do not pertain to any part of
104
+ the Derivative Works; and
105
+
106
+ (d) If the Work includes a "NOTICE" text file as part of its
107
+ distribution, then any Derivative Works that You distribute must
108
+ include a readable copy of the attribution notices contained
109
+ within such NOTICE file, excluding those notices that do not
110
+ pertain to any part of the Derivative Works, in at least one
111
+ of the following places: within a NOTICE text file distributed
112
+ as part of the Derivative Works; within the Source form or
113
+ documentation, if provided along with the Derivative Works; or,
114
+ within a display generated by the Derivative Works, if and
115
+ wherever such third-party notices normally appear. The contents
116
+ of the NOTICE file are for informational purposes only and
117
+ do not modify the License. You may add Your own attribution
118
+ notices within Derivative Works that You distribute, alongside
119
+ or as an addendum to the NOTICE text from the Work, provided
120
+ that such additional attribution notices cannot be construed
121
+ as modifying the License.
122
+
123
+ You may add Your own copyright statement to Your modifications and
124
+ may provide additional or different license terms and conditions
125
+ for use, reproduction, or distribution of Your modifications, or
126
+ for any such Derivative Works as a whole, provided Your use,
127
+ reproduction, and distribution of the Work otherwise complies with
128
+ the conditions stated in this License.
129
+
130
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
131
+ any Contribution intentionally submitted for inclusion in the Work
132
+ by You to the Licensor shall be under the terms and conditions of
133
+ this License, without any additional terms or conditions.
134
+ Notwithstanding the above, nothing herein shall supersede or modify
135
+ the terms of any separate license agreement you may have executed
136
+ with Licensor regarding such Contributions.
137
+
138
+ 6. Trademarks. This License does not grant permission to use the trade
139
+ names, trademarks, service marks, or product names of the Licensor,
140
+ except as required for reasonable and customary use in describing the
141
+ origin of the Work and reproducing the content of the NOTICE file.
142
+
143
+ 7. Disclaimer of Warranty. Unless required by applicable law or
144
+ agreed to in writing, Licensor provides the Work (and each
145
+ Contributor provides its Contributions) on an "AS IS" BASIS,
146
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147
+ implied, including, without limitation, any warranties or conditions
148
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149
+ PARTICULAR PURPOSE. You are solely responsible for determining the
150
+ appropriateness of using or redistributing the Work and assume any
151
+ risks associated with Your exercise of permissions under this License.
152
+
153
+ 8. Limitation of Liability. In no event and under no legal theory,
154
+ whether in tort (including negligence), contract, or otherwise,
155
+ unless required by applicable law (such as deliberate and grossly
156
+ negligent acts) or agreed to in writing, shall any Contributor be
157
+ liable to You for damages, including any direct, indirect, special,
158
+ incidental, or consequential damages of any character arising as a
159
+ result of this License or out of the use or inability to use the
160
+ Work (including but not limited to damages for loss of goodwill,
161
+ work stoppage, computer failure or malfunction, or any and all
162
+ other commercial damages or losses), even if such Contributor
163
+ has been advised of the possibility of such damages.
164
+
165
+ 9. Accepting Warranty or Additional Liability. While redistributing
166
+ the Work or Derivative Works thereof, You may choose to offer,
167
+ and charge a fee for, acceptance of support, warranty, indemnity,
168
+ or other liability obligations and/or rights consistent with this
169
+ License. However, in accepting such obligations, You may act only
170
+ on Your own behalf and on Your sole responsibility, not on behalf
171
+ of any other Contributor, and only if You agree to indemnify,
172
+ defend, and hold each Contributor harmless for any liability
173
+ incurred by, or claims asserted against, such Contributor by reason
174
+ of your accepting any such warranty or additional liability.
175
+
176
+ END OF TERMS AND CONDITIONS
177
+
178
+ APPENDIX: How to apply the Apache License to your work.
179
+
180
+ To apply the Apache License to your work, attach the following
181
+ boilerplate notice, with the fields enclosed by brackets "[]"
182
+ replaced with your own identifying information. (Don't include
183
+ the brackets!) The text should be enclosed in the appropriate
184
+ comment syntax for the file format. We also recommend that a
185
+ file or class name and description of purpose be included on the
186
+ same "printed page" as the copyright notice for easier
187
+ identification within third-party archives.
188
+
189
+ Copyright [yyyy] [name of copyright owner]
190
+
191
+ Licensed under the Apache License, Version 2.0 (the "License");
192
+ you may not use this file except in compliance with the License.
193
+ You may obtain a copy of the License at
194
+
195
+ http://www.apache.org/licenses/LICENSE-2.0
196
+
197
+ Unless required by applicable law or agreed to in writing, software
198
+ distributed under the License is distributed on an "AS IS" BASIS,
199
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
+ See the License for the specific language governing permissions and
201
+ limitations under the License.
package/README.md ADDED
@@ -0,0 +1,262 @@
1
+ # Google Cloud Datastore storage for Spine TS
2
+
3
+ `@spine-event-engine/storage-datastore` stores Spine TS records in Google Cloud
4
+ Datastore, including Firestore in Datastore mode. It implements the storage
5
+ contract from `@spine-event-engine/storage`; it does not use Firestore Native
6
+ APIs.
7
+
8
+ This is an experimental snapshot package. Use Node 24 or newer, generated
9
+ record schemas, and a configured Google Datastore client.
10
+
11
+ For exact query limits, entity-storage behavior, and failure handling, see
12
+ [REFERENCE documentation for agents](REFERENCE.md).
13
+
14
+ ## 💡 Why use it?
15
+
16
+ - ✅ Stores Spine records in Google Cloud Datastore or Firestore in Datastore mode.
17
+ - ✅ Pushes supported filters and ordering to Datastore.
18
+ - ✅ Preserves atomic compare-and-set writes through provider transactions.
19
+ - ✅ Rejects queries that exceed a finite reconciliation budget instead of
20
+ returning incomplete results.
21
+
22
+ ## 🚀 First snapshot success
23
+
24
+ ```sh
25
+ mkdir spine-datastore-app && cd spine-datastore-app
26
+ pnpm init
27
+ pnpm add @spine-event-engine/storage-datastore@snapshot @spine-event-engine/storage@snapshot @bufbuild/protobuf @google-cloud/datastore
28
+ ```
29
+
30
+ Create a `storage.ts` file from the factory example below, replace `my-project`
31
+ with a Google Cloud project that you can access, and run it with Node 24 or
32
+ newer. A successful read of `task-42` proves the installed snapshot can create,
33
+ write, and read a Datastore record. The snapshot tag can change before a stable
34
+ release.
35
+
36
+ ## 🔌 Create a factory
37
+
38
+ Give the adapter an already configured Datastore client when the application
39
+ controls Google authentication and client lifetime.
40
+
41
+ <!-- docs-snippet-path: packages/storage-datastore/src/index.ts -->
42
+
43
+ ```ts
44
+ import { Datastore } from "@google-cloud/datastore";
45
+ import { create, ScalarType } from "@bufbuild/protobuf";
46
+ import { StringValueSchema, type StringValue } from "@bufbuild/protobuf/wkt";
47
+ import { ColumnTypes, RecordColumn, RecordSpec } from "@spine-event-engine/storage";
48
+ import { DatastoreStorageFactory } from "@spine-event-engine/storage-datastore";
49
+
50
+ const factory = DatastoreStorageFactory.newBuilder()
51
+ .setClient(new Datastore({ projectId: "my-project" }))
52
+ .build();
53
+ const records = factory.createRecordStorage(
54
+ { name: "Tasks", multitenant: false },
55
+ new RecordSpec<string, StringValue>({
56
+ recordType: StringValueSchema,
57
+ idKind: "string",
58
+ extractId: (record) => record.value,
59
+ columns: [
60
+ new RecordColumn("value", ColumnTypes.scalar(ScalarType.STRING), (record) => record.value),
61
+ ],
62
+ }),
63
+ );
64
+
65
+ try {
66
+ await records.write(create(StringValueSchema, { value: "task-42" }));
67
+ const stored = await records.read("task-42");
68
+ if (stored?.value !== "task-42") throw new Error("The record was not stored.");
69
+ } finally {
70
+ await records.close();
71
+ factory.close();
72
+ }
73
+ ```
74
+
75
+ The builder always uses a client supplied by the caller. Its fixed finite reconciliation
76
+ bound is 1,000 records. Pass the factory to the server or create record storage
77
+ through the normal storage API. When application values contain `Any`, configure
78
+ a `StringifierRegistry` with that application's `TypeRegistry` before building;
79
+ the layout section below explains why that mapping must remain reversible.
80
+
81
+ ## Build it in this workspace
82
+
83
+ Contributors changing this repository can run the workspace-wide build from its
84
+ root:
85
+
86
+ ```sh
87
+ pnpm typecheck:build
88
+ ```
89
+
90
+ ## 🧭 Understand tenants and kinds
91
+
92
+ Datastore uses its native namespace as the tenant boundary. By default, Spine
93
+ converts a complete generated `TenantId` to the safe reversible Spine JVM forms:
94
+ `Dexample.org` for a domain or `Vtenant-a` for a plain value. Spine JVM's email
95
+ form replaces `@` with `-at-`; two different emails can therefore become the
96
+ same namespace. Spine TS rejects that unsafe default. To use email tenant IDs,
97
+ install the same reversible custom namespace converter in both runtimes.
98
+ Single-tenant storage keeps the namespace already configured on the
99
+ caller-supplied client. A Bounded Context name helps diagnostics; it never changes
100
+ a namespace, kind, or key.
101
+
102
+ Suppose a projection state is declared like this:
103
+
104
+ ```proto
105
+ package spine.examples.messageboard;
106
+
107
+ message MessageView {
108
+ MessageId id = 1 [(required) = true, (set_once) = true];
109
+ BoardId board = 2 [(column) = true];
110
+ UserId author = 3 [(column) = true];
111
+ string text = 4;
112
+ }
113
+ ```
114
+
115
+ With the default layout, its kind is the full Proto type name, for example
116
+ `spine.examples.messageboard.MessageView`. One entity looks conceptually like
117
+ this:
118
+
119
+ | Datastore part | Stored value |
120
+ | -------------- | ----------------------------------------- |
121
+ | namespace | `Vtenant-a` |
122
+ | kind | `spine.examples.messageboard.MessageView` |
123
+ | key name | `{"value":"message-42"}` |
124
+ | `bytes` | unindexed `EntityRecord` with the state |
125
+ | `archived` | native boolean |
126
+ | `deleted` | native boolean |
127
+ | `version` | native integer |
128
+ | `board` | `{"value":"board-7"}` |
129
+ | `author` | `{"value":"user-3"}` |
130
+
131
+ `archived`, `deleted`, and `version` come from Spine's generated `EntityRecord`;
132
+ they are Entity lifecycle/version facts, not a provider revision. `board` and
133
+ `author` exist because the Proto fields use `(column)`. The unindexed `bytes`
134
+ payload remains authoritative.
135
+
136
+ The same identity and value mapping is used at each step: `Identifiers` turn
137
+ the record ID into the key name, while the configured `StringifierRegistry`
138
+ turns message-valued IDs and `(column)` values into reversible text. Do not
139
+ pre-stringify a query value in application code. `RecordQuery<I>` statically
140
+ types IDs only; Datastore checks the string filter name and `unknown` value
141
+ against the record descriptor and column mapping, then applies the registry
142
+ mapping used on write.
143
+
144
+ Message-valued IDs and columns use compact Proto JSON by default. Supply the
145
+ application `TypeRegistry` as shown above when a stored framework value contains
146
+ an `Any`; it tells Proto JSON how to expand the packed application type.
147
+ Applications can also register one reversible custom `Stringifier`. The same
148
+ mapping is used for writes, key lookups, filters, ordering continuations, and
149
+ reads. Primitive values use their native Datastore representation. There is no
150
+ `_scope`, copied ID, storage revision, fingerprint, marker, or compatibility
151
+ entity. The runnable configuration is in the
152
+ [Message Board deployment](https://github.com/SpineEventEngine/spine-ts/blob/master/examples/message-board/app/src/deployment-config.ts).
153
+
154
+ When a user queries `board == BoardId("board-7")`, Spine converts that generated
155
+ `BoardId` with the same stringifier and sends a Datastore property filter for
156
+ `board == '{"value":"board-7"}'`. Datastore uses the declared property and its
157
+ indexes to find matching keys; Spine decodes the authoritative `bytes` payload
158
+ for each result.
159
+
160
+ ## 🔐 Configure credentials and indexes
161
+
162
+ Credential selection belongs to the official Google client. For example, an
163
+ application can use Application Default Credentials or pass `credentials` or
164
+ `keyFilename` in its Datastore options. Keep credentials out of source code and
165
+ logs.
166
+
167
+ Queries that combine equality filters and ordering can require composite
168
+ indexes. Kind names depend on the application's Proto types and layout choices,
169
+ so the framework cannot ship one universal index file. Keep the required
170
+ indexes in the application and deploy them before serving those queries. For
171
+ example:
172
+
173
+ ```yaml
174
+ indexes:
175
+ - kind: spine.examples.orders.OrderView
176
+ properties:
177
+ - name: status
178
+ - name: when_created
179
+ direction: desc
180
+ ```
181
+
182
+ ```sh
183
+ gcloud datastore indexes create index.yaml
184
+ gcloud datastore indexes list
185
+ ```
186
+
187
+ Wait until each required index is ready before serving its query combinations.
188
+
189
+ Retained Entity histories need separate indexes because their kinds and
190
+ ordering differ from current state. For a state type named
191
+ `spine.examples.board.Message`, the usual backward-history index is:
192
+
193
+ ```yaml
194
+ indexes:
195
+ - kind: spine.examples.board.Message_EntityRecord
196
+ properties:
197
+ - name: entity_id
198
+ - name: version
199
+ direction: desc
200
+ - name: created
201
+ direction: desc
202
+ ```
203
+
204
+ `stateAt()` instead orders `created` and then `version`, both descending.
205
+ Diagnostic event history uses the corresponding
206
+ `<state-source>_Event` grouped kind and the same `entity_id`, `version`, and
207
+ `created` columns. Inspect the concrete query shapes your application serves
208
+ and add the matching Datastore composite indexes; the adapter never creates
209
+ them automatically.
210
+
211
+ ## 📏 Understand query limits
212
+
213
+ The adapter has a fixed finite client-side reconciliation budget of `1000`.
214
+ When a query needs more candidates than that budget,
215
+ `DatastoreQueryLimitError` is thrown instead of returning a partial result.
216
+ There is no unlimited scan setting or Datastore-specific cursor API.
217
+ Provider filters and ordering are pushed down; public query offsets and mixed
218
+ local/provider limits are reconciled locally within that finite bound.
219
+
220
+ Ordinary `write()` and `writeAll()` calls are independent provider writes. Use
221
+ compare-and-set for an atomic conditional record update, or an Entity commit
222
+ for one atomic current-state/history/event mutation.
223
+
224
+ ## 🧪 Verify with a local emulator
225
+
226
+ The optional emulator suite needs an already-running Firestore emulator in
227
+ Datastore mode.
228
+
229
+ ```sh
230
+ gcloud emulators firestore start --database-mode=datastore-mode --host-port=127.0.0.1:8081
231
+ DATASTORE_EMULATOR_HOST=127.0.0.1:8081 \
232
+ pnpm --filter @spine-event-engine/storage-datastore test:emulator
233
+ ```
234
+
235
+ Use a disposable project for the credential-gated cloud smoke test. Neither
236
+ test substitutes for production index, quota, or consistency planning.
237
+
238
+ ## ⚠️ Before production
239
+
240
+ Deploy every composite index used by the application, select quotas and retry
241
+ policy for its workload, and keep Google credentials outside source control.
242
+ This adapter does not provide an unlimited scan mode or a Datastore-specific
243
+ cursor API.
244
+
245
+ Before starting this corrected layout in an existing project, inventory every
246
+ native namespace and kind:
247
+
248
+ ```sh
249
+ pnpm --dir packages/storage-datastore inventory:legacy -- --project my-project
250
+ ```
251
+
252
+ The command exits nonzero if discovery fails or it finds an old `_scope`
253
+ property or scope-derived key name. Do not start the corrected runtime until it
254
+ passes. Old rows are invisible to direct keys; move them offline, and stop if
255
+ rows from old Bounded Context partitions would become the same namespace,
256
+ kind, and key. The runtime never chooses a winner.
257
+
258
+ ## 🔗 Learn more
259
+
260
+ - [Storage API](https://github.com/SpineEventEngine/spine-ts/blob/master/packages/storage/README.md)
261
+ - [Datastore guide](https://github.com/SpineEventEngine/spine-ts/blob/master/docs/USER_GUIDE.md#6-persist-application-data)
262
+ - [Reference for coding agents](REFERENCE.md)
package/REFERENCE.md ADDED
@@ -0,0 +1,147 @@
1
+ # @spine-event-engine/storage-datastore reference
2
+
3
+ This reference is for agents configuring the Google Cloud Datastore adapter.
4
+
5
+ ## Public entry point
6
+
7
+ Import `DatastoreStorageFactory`, `DatastoreStorageFactoryBuilder`,
8
+ `RecordLayout`, `CreateRecordStorage`, `CreateEntityStorage`,
9
+ `NamespaceConverter`, `DefaultNamespaceConverter`, `DatastoreIdColumn`,
10
+ `DatastoreColumnMapping`, and `DatastoreQueryLimitError` from
11
+ `@spine-event-engine/storage-datastore`.
12
+ Create a factory with `DatastoreStorageFactory.newBuilder().setClient(client)`.
13
+ The adapter never closes that client.
14
+
15
+ The builder is mutable until `build()` and the built factory receives a
16
+ snapshot. The last registration for the same identity wins. Record resolution
17
+ uses this precedence:
18
+
19
+ 1. an exact source-plus-record creator;
20
+ 2. a record-only creator;
21
+ 3. an exact source-plus-record layout;
22
+ 4. a record-only layout; and
23
+ 5. the default kind.
24
+
25
+ `setNamespaceConverter(...)` replaces complete `TenantId`/native namespace
26
+ conversion. `setStringifierRegistry(...)` supplies reversible mappings for
27
+ message-valued IDs and ordinary message columns. The stringifier registry is
28
+ snapshotted by `build()`; a custom namespace converter is retained and must
29
+ remain behaviorally immutable for the factory lifetime. `organizeRecords(...)`
30
+ changes a kind. `useRecordStorage(...)` replaces a record-family provider.
31
+ `Identifiers` select the key representation, and the same configured
32
+ stringifier maps a message-valued ID or declared column on write, direct lookup,
33
+ filter, ordering continuation, and read. `RecordQuery<I>` statically types IDs
34
+ only. Datastore maps filter values through its record column mapping at runtime;
35
+ its sort-property mapping is the requested field name. Callers do not construct
36
+ a provider string or infer shared filter/sort-name validation.
37
+ `useEntityStorage(...)` replaces the complete coherent Entity handle, including
38
+ its commit capability. Custom providers receive the storage context, record or
39
+ Entity contract, and caller-supplied client, but not the built-in converter or
40
+ stringifier registry; they must provide equivalent tenant, ID, and column mapping. The
41
+ built-in Entity provider does not mix custom record creators into one transaction.
42
+
43
+ ## Physical layout
44
+
45
+ Every record family uses the source Proto full name as its default kind. A
46
+ grouped family uses `<group>_<record-simple-name>`. A layout registration can
47
+ replace only the kind. Rows contain unindexed Protobuf `bytes` and the declared
48
+ indexed columns. Current Entity rows declare `archived`, `deleted`, and Entity
49
+ `version` in addition to model `(column)` fields; Entity `version` is not a
50
+ provider revision. No scope, ID copy, storage revision, schema fingerprint,
51
+ marker, or compatibility entity is stored. The Bounded Context name is
52
+ diagnostic only.
53
+
54
+ Single tenancy preserves the caller client's configured/default namespace.
55
+ Multitenancy converts the complete generated `TenantId` to a native namespace.
56
+ The default converter supports the reversible JVM `D<domain>` and `V<value>`
57
+ forms. It rejects email tenants because JVM's `E` conversion replaces `@` with
58
+ `-at-` and is not injective. Email tenants require one reversible custom
59
+ converter installed in both TS and JVM. Catalog admission rejects empty,
60
+ non-round-tripping, or colliding converter results before provider work. The
61
+ key contains only the resolved kind and mapped record ID. Message IDs use
62
+ compact Proto JSON or their
63
+ registered custom stringifier; `string`, `int32`, and `int64` IDs use their
64
+ direct text form as the key name. `bytes` and `__key__` are reserved. Blank
65
+ kinds, kinds or key names over 1,500 UTF-8 bytes, unsupported values, non-finite
66
+ numbers, and integers outside their declared/provider range fail before an
67
+ RPC. Stored `bytes` remain authoritative; declared properties are rematerialized
68
+ after decoding.
69
+
70
+ `keep()` bridges native `__namespace__` metadata lag without storing a tenant
71
+ record. Its early-admission cache retains at most 1,000 tenants for at most 60
72
+ seconds and removes an entry sooner when native metadata becomes visible. A
73
+ full cache fails closed; applications can retry after metadata appears or an
74
+ entry expires.
75
+
76
+ Declared string and boolean columns use native values. Integer columns use
77
+ Datastore int values, float/double columns use Datastore double values, bytes
78
+ use blobs, enums use their numeric value, Timestamp uses the native timestamp,
79
+ Version uses its number, and ordinary messages use the same compact Proto JSON
80
+ or custom stringifier as query operands. Null is stored as native null. The
81
+ application type registry is required when default Proto JSON expands `Any`.
82
+
83
+ Entity current records use `EntityRecord`; enabled state history uses grouped
84
+ `EntityStateKey`/`EntityRecord`; diagnostic history uses grouped
85
+ `EventId`/`Event`; Event Store is a separate ungrouped Event family. Disabled
86
+ histories allocate no record handle or Datastore row.
87
+
88
+ ## Queries and commits
89
+
90
+ Queries run inside the selected native namespace. Normalized plans admit only
91
+ IDs (up to Datastore's legal key filter bound), equality, one inequality column,
92
+ flat conjunction, inequality-compatible ordering, limit, and mask. Nested or
93
+ disjunctive predicates, oversized key sets, and illegal inequality/order shapes
94
+ reject before provider access; they never trigger an unfiltered reconciliation
95
+ read.
96
+ `writeAll()` uses batches of at most 500 mutations and is not atomic across
97
+ batches.
98
+
99
+ Provider-legal ID predicates, declared-property comparisons, and ordering are
100
+ pushed only when Datastore can execute the whole selected conjunction. Runtime
101
+ descriptor/column validation happens before that decision. An omitted query
102
+ budget uses the shared 10,000-record materialization bound, but Datastore's
103
+ explicit value cannot exceed 10,000. Datastore's accepted/provider ceiling is
104
+ 1,000 rows. It may read one overflow-lookahead row
105
+ (1,001 raw rows) to reject an oversized scan rather than materializing more
106
+ rows. There is no public Datastore cursor API.
107
+
108
+ This is deliberately the overlap, not MySQL parity: nested or disjunctive
109
+ predicates and provider-illegal inequality/order shapes reject before provider
110
+ access. Normalized plans have no offset; `RecordQuery.offset` is separate.
111
+
112
+ The internal Entity commit reads current and immutable keys then applies current,
113
+ enabled histories, and delivery events in one Datastore transaction. It rejects
114
+ more than 25 entity groups, 500 mutations, or its conservative transaction-size
115
+ limit before opening the transaction. Current mismatch returns `conflict`.
116
+ An already-applied identical retry returns `committed`; divergent immutable
117
+ content fails. Only ABORTED provider failures retry, for at most three attempts.
118
+
119
+ History reads use stable finite keyset pages. State history provides backward,
120
+ state-at-time, trim, and truncate behavior; event history provides backward and
121
+ truncate behavior. Timestamp comparisons include seconds and nanoseconds.
122
+ Long maintenance can commit several bounded chunks, so a later failure leaves
123
+ earlier chunks durable and the caller retries the same idempotent operation.
124
+
125
+ The factory has one tenant catalog. It reads native `__namespace__` metadata,
126
+ converts only namespaces recognized by its `NamespaceConverter`, and keeps an early
127
+ in-memory cache for newly admitted tenants. `keep()` stores no `TenantId` row or
128
+ other discovery record; Datastore exposes native metadata after an application
129
+ entity is written in that namespace.
130
+
131
+ Before deployment to an existing project, run `pnpm --dir
132
+ packages/storage-datastore inventory:legacy -- --project <project-id>`. It
133
+ enumerates native namespaces and kinds and fails closed on discovery errors,
134
+ an old `_scope` property, or a scope-derived key name. Passing the inventory is
135
+ a startup prerequisite; the application performs migration offline, with no
136
+ dual-layout reads or automatic conflict winner.
137
+
138
+ ## Operations and errors
139
+
140
+ Malformed payloads fail with a redacted decoding error. Physical Datastore
141
+ errors are surfaced without payload data. The emulator suite is required for
142
+ provider acceptance; cloud smoke remains credential-gated.
143
+
144
+ `build()` and storage-handle creation issue no request. Closing the factory is
145
+ idempotent and prevents creation of another handle without closing the
146
+ caller-supplied client or already-created handles. Each record, Entity, and commit
147
+ handle has an independent idempotent lifecycle and rejects work after it is closed.
@@ -0,0 +1,28 @@
1
+ import { StringifierRegistry } from "@spine-event-engine/core";
2
+ import type { ColumnMapping, ColumnTypeMapping, RecordColumnType } from "@spine-event-engine/storage";
3
+ /**
4
+ * Converts typed Protobuf column values to Spine JVM Datastore values.
5
+ */
6
+ export declare class DatastoreColumnMapping implements ColumnMapping<unknown> {
7
+ #private;
8
+ /**
9
+ * Creates a JVM-compatible Datastore column mapping.
10
+ *
11
+ * @param stringifiers The schema-bound message stringifiers.
12
+ */
13
+ constructor(stringifiers?: StringifierRegistry);
14
+ /**
15
+ * Returns the conversion for one generated column type.
16
+ *
17
+ * @param type The generated Protobuf column type.
18
+ * @returns The JVM-compatible Datastore conversion.
19
+ */
20
+ of<V>(type: RecordColumnType<V>): ColumnTypeMapping<V, unknown>;
21
+ /**
22
+ * Returns native Datastore null.
23
+ *
24
+ * @returns The null conversion.
25
+ */
26
+ ofNull(): ColumnTypeMapping<null, unknown>;
27
+ }
28
+ //# sourceMappingURL=column-mapping.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"column-mapping.d.ts","sourceRoot":"","sources":["../../src/datastore/column-mapping.ts"],"names":[],"mappings":"AAiBA,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAC/D,OAAO,KAAK,EACV,aAAa,EACb,iBAAiB,EACjB,gBAAgB,EACjB,MAAM,6BAA6B,CAAC;AAKrC;;GAEG;AACH,qBAAa,sBAAuB,YAAW,aAAa,CAAC,OAAO,CAAC;;IAGnE;;;;OAIG;gBACS,YAAY,GAAE,mBAA+C;IAIzE;;;;;OAKG;IACH,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,gBAAgB,CAAC,CAAC,CAAC,GAAG,iBAAiB,CAAC,CAAC,EAAE,OAAO,CAAC;IAkB/D;;;;OAIG;IACH,MAAM,IAAI,iBAAiB,CAAC,IAAI,EAAE,OAAO,CAAC;CAG3C"}