@spine-event-engine/storage-datastore 2.0.0-snapshot.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +262 -0
- package/REFERENCE.md +147 -0
- package/dist/datastore/column-mapping.d.ts +28 -0
- package/dist/datastore/column-mapping.d.ts.map +1 -0
- package/dist/datastore/column-mapping.js +141 -0
- package/dist/datastore/column-mapping.js.map +1 -0
- package/dist/datastore/delivery-cleanup.d.ts +31 -0
- package/dist/datastore/delivery-cleanup.d.ts.map +1 -0
- package/dist/datastore/delivery-cleanup.js +111 -0
- package/dist/datastore/delivery-cleanup.js.map +1 -0
- package/dist/datastore/entity-history.d.ts +82 -0
- package/dist/datastore/entity-history.d.ts.map +1 -0
- package/dist/datastore/entity-history.js +541 -0
- package/dist/datastore/entity-history.js.map +1 -0
- package/dist/datastore/id-column.d.ts +34 -0
- package/dist/datastore/id-column.d.ts.map +1 -0
- package/dist/datastore/id-column.js +103 -0
- package/dist/datastore/id-column.js.map +1 -0
- package/dist/datastore/namespace.d.ts +81 -0
- package/dist/datastore/namespace.d.ts.map +1 -0
- package/dist/datastore/namespace.js +123 -0
- package/dist/datastore/namespace.js.map +1 -0
- package/dist/datastore/record-storage.d.ts +223 -0
- package/dist/datastore/record-storage.d.ts.map +1 -0
- package/dist/datastore/record-storage.js +855 -0
- package/dist/datastore/record-storage.js.map +1 -0
- package/dist/datastore/storage-factory.d.ts +200 -0
- package/dist/datastore/storage-factory.d.ts.map +1 -0
- package/dist/datastore/storage-factory.js +224 -0
- package/dist/datastore/storage-factory.js.map +1 -0
- package/dist/datastore/tenant-catalog.d.ts +50 -0
- package/dist/datastore/tenant-catalog.d.ts.map +1 -0
- package/dist/datastore/tenant-catalog.js +154 -0
- package/dist/datastore/tenant-catalog.js.map +1 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +19 -0
- package/dist/index.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -0
- package/index.yaml +1 -0
- package/package.json +40 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
6
|
+
|
|
7
|
+
1. Definitions.
|
|
8
|
+
|
|
9
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
10
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
11
|
+
|
|
12
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
13
|
+
the copyright owner that is granting the License.
|
|
14
|
+
|
|
15
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
16
|
+
other entities that control, are controlled by, or are under common
|
|
17
|
+
control with that entity. For the purposes of this definition,
|
|
18
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
19
|
+
direction or management of such entity, whether by contract or
|
|
20
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
21
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
22
|
+
|
|
23
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
24
|
+
exercising permissions granted by this License.
|
|
25
|
+
|
|
26
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
27
|
+
including but not limited to software source code, documentation
|
|
28
|
+
source, and configuration files.
|
|
29
|
+
|
|
30
|
+
"Object" form shall mean any form resulting from mechanical
|
|
31
|
+
transformation or translation of a Source form, including but
|
|
32
|
+
not limited to compiled object code, generated documentation,
|
|
33
|
+
and conversions to other media types.
|
|
34
|
+
|
|
35
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
36
|
+
Object form, made available under the License, as indicated by a
|
|
37
|
+
copyright notice that is included in or attached to the work
|
|
38
|
+
(an example is provided in the Appendix below).
|
|
39
|
+
|
|
40
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
41
|
+
form, that is based on (or derived from) the Work and for which the
|
|
42
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
43
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
44
|
+
of this License, Derivative Works shall not include works that remain
|
|
45
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
46
|
+
the Work and Derivative Works thereof.
|
|
47
|
+
|
|
48
|
+
"Contribution" shall mean any work of authorship, including
|
|
49
|
+
the original version of the Work and any modifications or additions
|
|
50
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
51
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
52
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
53
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
54
|
+
means any form of electronic, verbal, or written communication sent
|
|
55
|
+
to the Licensor or its representatives, including but not limited to
|
|
56
|
+
communication on electronic mailing lists, source code control systems,
|
|
57
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
58
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
59
|
+
excluding communication that is conspicuously marked or otherwise
|
|
60
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
61
|
+
|
|
62
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
63
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
64
|
+
subsequently incorporated within the Work.
|
|
65
|
+
|
|
66
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
67
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
68
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
69
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
70
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
71
|
+
Work and such Derivative Works in Source or Object form.
|
|
72
|
+
|
|
73
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
74
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
75
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
76
|
+
(except as stated in this section) patent license to make, have made,
|
|
77
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
78
|
+
where such license applies only to those patent claims licensable
|
|
79
|
+
by such Contributor that are necessarily infringed by their
|
|
80
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
81
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
82
|
+
institute patent litigation against any entity (including a
|
|
83
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
84
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
85
|
+
or contributory patent infringement, then any patent licenses
|
|
86
|
+
granted to You under this License for that Work shall terminate
|
|
87
|
+
as of the date such litigation is filed.
|
|
88
|
+
|
|
89
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
90
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
91
|
+
modifications, and in Source or Object form, provided that You
|
|
92
|
+
meet the following conditions:
|
|
93
|
+
|
|
94
|
+
(a) You must give any other recipients of the Work or
|
|
95
|
+
Derivative Works a copy of this License; and
|
|
96
|
+
|
|
97
|
+
(b) You must cause any modified files to carry prominent notices
|
|
98
|
+
stating that You changed the files; and
|
|
99
|
+
|
|
100
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
101
|
+
that You distribute, all copyright, patent, trademark, and
|
|
102
|
+
attribution notices from the Source form of the Work,
|
|
103
|
+
excluding those notices that do not pertain to any part of
|
|
104
|
+
the Derivative Works; and
|
|
105
|
+
|
|
106
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
107
|
+
distribution, then any Derivative Works that You distribute must
|
|
108
|
+
include a readable copy of the attribution notices contained
|
|
109
|
+
within such NOTICE file, excluding those notices that do not
|
|
110
|
+
pertain to any part of the Derivative Works, in at least one
|
|
111
|
+
of the following places: within a NOTICE text file distributed
|
|
112
|
+
as part of the Derivative Works; within the Source form or
|
|
113
|
+
documentation, if provided along with the Derivative Works; or,
|
|
114
|
+
within a display generated by the Derivative Works, if and
|
|
115
|
+
wherever such third-party notices normally appear. The contents
|
|
116
|
+
of the NOTICE file are for informational purposes only and
|
|
117
|
+
do not modify the License. You may add Your own attribution
|
|
118
|
+
notices within Derivative Works that You distribute, alongside
|
|
119
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
120
|
+
that such additional attribution notices cannot be construed
|
|
121
|
+
as modifying the License.
|
|
122
|
+
|
|
123
|
+
You may add Your own copyright statement to Your modifications and
|
|
124
|
+
may provide additional or different license terms and conditions
|
|
125
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
126
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
127
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
128
|
+
the conditions stated in this License.
|
|
129
|
+
|
|
130
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
131
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
132
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
133
|
+
this License, without any additional terms or conditions.
|
|
134
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
135
|
+
the terms of any separate license agreement you may have executed
|
|
136
|
+
with Licensor regarding such Contributions.
|
|
137
|
+
|
|
138
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
139
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
140
|
+
except as required for reasonable and customary use in describing the
|
|
141
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
142
|
+
|
|
143
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
144
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
145
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
146
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
147
|
+
implied, including, without limitation, any warranties or conditions
|
|
148
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
149
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
150
|
+
appropriateness of using or redistributing the Work and assume any
|
|
151
|
+
risks associated with Your exercise of permissions under this License.
|
|
152
|
+
|
|
153
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
154
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
155
|
+
unless required by applicable law (such as deliberate and grossly
|
|
156
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
157
|
+
liable to You for damages, including any direct, indirect, special,
|
|
158
|
+
incidental, or consequential damages of any character arising as a
|
|
159
|
+
result of this License or out of the use or inability to use the
|
|
160
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
161
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
162
|
+
other commercial damages or losses), even if such Contributor
|
|
163
|
+
has been advised of the possibility of such damages.
|
|
164
|
+
|
|
165
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
166
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
167
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
168
|
+
or other liability obligations and/or rights consistent with this
|
|
169
|
+
License. However, in accepting such obligations, You may act only
|
|
170
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
171
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
172
|
+
defend, and hold each Contributor harmless for any liability
|
|
173
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
174
|
+
of your accepting any such warranty or additional liability.
|
|
175
|
+
|
|
176
|
+
END OF TERMS AND CONDITIONS
|
|
177
|
+
|
|
178
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
179
|
+
|
|
180
|
+
To apply the Apache License to your work, attach the following
|
|
181
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
182
|
+
replaced with your own identifying information. (Don't include
|
|
183
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
184
|
+
comment syntax for the file format. We also recommend that a
|
|
185
|
+
file or class name and description of purpose be included on the
|
|
186
|
+
same "printed page" as the copyright notice for easier
|
|
187
|
+
identification within third-party archives.
|
|
188
|
+
|
|
189
|
+
Copyright [yyyy] [name of copyright owner]
|
|
190
|
+
|
|
191
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
|
+
you may not use this file except in compliance with the License.
|
|
193
|
+
You may obtain a copy of the License at
|
|
194
|
+
|
|
195
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
196
|
+
|
|
197
|
+
Unless required by applicable law or agreed to in writing, software
|
|
198
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
199
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
200
|
+
See the License for the specific language governing permissions and
|
|
201
|
+
limitations under the License.
|
package/README.md
ADDED
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
# Google Cloud Datastore storage for Spine TS
|
|
2
|
+
|
|
3
|
+
`@spine-event-engine/storage-datastore` stores Spine TS records in Google Cloud
|
|
4
|
+
Datastore, including Firestore in Datastore mode. It implements the storage
|
|
5
|
+
contract from `@spine-event-engine/storage`; it does not use Firestore Native
|
|
6
|
+
APIs.
|
|
7
|
+
|
|
8
|
+
This is an experimental snapshot package. Use Node 24 or newer, generated
|
|
9
|
+
record schemas, and a configured Google Datastore client.
|
|
10
|
+
|
|
11
|
+
For exact query limits, entity-storage behavior, and failure handling, see
|
|
12
|
+
[REFERENCE documentation for agents](REFERENCE.md).
|
|
13
|
+
|
|
14
|
+
## 💡 Why use it?
|
|
15
|
+
|
|
16
|
+
- ✅ Stores Spine records in Google Cloud Datastore or Firestore in Datastore mode.
|
|
17
|
+
- ✅ Pushes supported filters and ordering to Datastore.
|
|
18
|
+
- ✅ Preserves atomic compare-and-set writes through provider transactions.
|
|
19
|
+
- ✅ Rejects queries that exceed a finite reconciliation budget instead of
|
|
20
|
+
returning incomplete results.
|
|
21
|
+
|
|
22
|
+
## 🚀 First snapshot success
|
|
23
|
+
|
|
24
|
+
```sh
|
|
25
|
+
mkdir spine-datastore-app && cd spine-datastore-app
|
|
26
|
+
pnpm init
|
|
27
|
+
pnpm add @spine-event-engine/storage-datastore@snapshot @spine-event-engine/storage@snapshot @bufbuild/protobuf @google-cloud/datastore
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Create a `storage.ts` file from the factory example below, replace `my-project`
|
|
31
|
+
with a Google Cloud project that you can access, and run it with Node 24 or
|
|
32
|
+
newer. A successful read of `task-42` proves the installed snapshot can create,
|
|
33
|
+
write, and read a Datastore record. The snapshot tag can change before a stable
|
|
34
|
+
release.
|
|
35
|
+
|
|
36
|
+
## 🔌 Create a factory
|
|
37
|
+
|
|
38
|
+
Give the adapter an already configured Datastore client when the application
|
|
39
|
+
controls Google authentication and client lifetime.
|
|
40
|
+
|
|
41
|
+
<!-- docs-snippet-path: packages/storage-datastore/src/index.ts -->
|
|
42
|
+
|
|
43
|
+
```ts
|
|
44
|
+
import { Datastore } from "@google-cloud/datastore";
|
|
45
|
+
import { create, ScalarType } from "@bufbuild/protobuf";
|
|
46
|
+
import { StringValueSchema, type StringValue } from "@bufbuild/protobuf/wkt";
|
|
47
|
+
import { ColumnTypes, RecordColumn, RecordSpec } from "@spine-event-engine/storage";
|
|
48
|
+
import { DatastoreStorageFactory } from "@spine-event-engine/storage-datastore";
|
|
49
|
+
|
|
50
|
+
const factory = DatastoreStorageFactory.newBuilder()
|
|
51
|
+
.setClient(new Datastore({ projectId: "my-project" }))
|
|
52
|
+
.build();
|
|
53
|
+
const records = factory.createRecordStorage(
|
|
54
|
+
{ name: "Tasks", multitenant: false },
|
|
55
|
+
new RecordSpec<string, StringValue>({
|
|
56
|
+
recordType: StringValueSchema,
|
|
57
|
+
idKind: "string",
|
|
58
|
+
extractId: (record) => record.value,
|
|
59
|
+
columns: [
|
|
60
|
+
new RecordColumn("value", ColumnTypes.scalar(ScalarType.STRING), (record) => record.value),
|
|
61
|
+
],
|
|
62
|
+
}),
|
|
63
|
+
);
|
|
64
|
+
|
|
65
|
+
try {
|
|
66
|
+
await records.write(create(StringValueSchema, { value: "task-42" }));
|
|
67
|
+
const stored = await records.read("task-42");
|
|
68
|
+
if (stored?.value !== "task-42") throw new Error("The record was not stored.");
|
|
69
|
+
} finally {
|
|
70
|
+
await records.close();
|
|
71
|
+
factory.close();
|
|
72
|
+
}
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
The builder always uses a client supplied by the caller. Its fixed finite reconciliation
|
|
76
|
+
bound is 1,000 records. Pass the factory to the server or create record storage
|
|
77
|
+
through the normal storage API. When application values contain `Any`, configure
|
|
78
|
+
a `StringifierRegistry` with that application's `TypeRegistry` before building;
|
|
79
|
+
the layout section below explains why that mapping must remain reversible.
|
|
80
|
+
|
|
81
|
+
## Build it in this workspace
|
|
82
|
+
|
|
83
|
+
Contributors changing this repository can run the workspace-wide build from its
|
|
84
|
+
root:
|
|
85
|
+
|
|
86
|
+
```sh
|
|
87
|
+
pnpm typecheck:build
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
## 🧭 Understand tenants and kinds
|
|
91
|
+
|
|
92
|
+
Datastore uses its native namespace as the tenant boundary. By default, Spine
|
|
93
|
+
converts a complete generated `TenantId` to the safe reversible Spine JVM forms:
|
|
94
|
+
`Dexample.org` for a domain or `Vtenant-a` for a plain value. Spine JVM's email
|
|
95
|
+
form replaces `@` with `-at-`; two different emails can therefore become the
|
|
96
|
+
same namespace. Spine TS rejects that unsafe default. To use email tenant IDs,
|
|
97
|
+
install the same reversible custom namespace converter in both runtimes.
|
|
98
|
+
Single-tenant storage keeps the namespace already configured on the
|
|
99
|
+
caller-supplied client. A Bounded Context name helps diagnostics; it never changes
|
|
100
|
+
a namespace, kind, or key.
|
|
101
|
+
|
|
102
|
+
Suppose a projection state is declared like this:
|
|
103
|
+
|
|
104
|
+
```proto
|
|
105
|
+
package spine.examples.messageboard;
|
|
106
|
+
|
|
107
|
+
message MessageView {
|
|
108
|
+
MessageId id = 1 [(required) = true, (set_once) = true];
|
|
109
|
+
BoardId board = 2 [(column) = true];
|
|
110
|
+
UserId author = 3 [(column) = true];
|
|
111
|
+
string text = 4;
|
|
112
|
+
}
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
With the default layout, its kind is the full Proto type name, for example
|
|
116
|
+
`spine.examples.messageboard.MessageView`. One entity looks conceptually like
|
|
117
|
+
this:
|
|
118
|
+
|
|
119
|
+
| Datastore part | Stored value |
|
|
120
|
+
| -------------- | ----------------------------------------- |
|
|
121
|
+
| namespace | `Vtenant-a` |
|
|
122
|
+
| kind | `spine.examples.messageboard.MessageView` |
|
|
123
|
+
| key name | `{"value":"message-42"}` |
|
|
124
|
+
| `bytes` | unindexed `EntityRecord` with the state |
|
|
125
|
+
| `archived` | native boolean |
|
|
126
|
+
| `deleted` | native boolean |
|
|
127
|
+
| `version` | native integer |
|
|
128
|
+
| `board` | `{"value":"board-7"}` |
|
|
129
|
+
| `author` | `{"value":"user-3"}` |
|
|
130
|
+
|
|
131
|
+
`archived`, `deleted`, and `version` come from Spine's generated `EntityRecord`;
|
|
132
|
+
they are Entity lifecycle/version facts, not a provider revision. `board` and
|
|
133
|
+
`author` exist because the Proto fields use `(column)`. The unindexed `bytes`
|
|
134
|
+
payload remains authoritative.
|
|
135
|
+
|
|
136
|
+
The same identity and value mapping is used at each step: `Identifiers` turn
|
|
137
|
+
the record ID into the key name, while the configured `StringifierRegistry`
|
|
138
|
+
turns message-valued IDs and `(column)` values into reversible text. Do not
|
|
139
|
+
pre-stringify a query value in application code. `RecordQuery<I>` statically
|
|
140
|
+
types IDs only; Datastore checks the string filter name and `unknown` value
|
|
141
|
+
against the record descriptor and column mapping, then applies the registry
|
|
142
|
+
mapping used on write.
|
|
143
|
+
|
|
144
|
+
Message-valued IDs and columns use compact Proto JSON by default. Supply the
|
|
145
|
+
application `TypeRegistry` as shown above when a stored framework value contains
|
|
146
|
+
an `Any`; it tells Proto JSON how to expand the packed application type.
|
|
147
|
+
Applications can also register one reversible custom `Stringifier`. The same
|
|
148
|
+
mapping is used for writes, key lookups, filters, ordering continuations, and
|
|
149
|
+
reads. Primitive values use their native Datastore representation. There is no
|
|
150
|
+
`_scope`, copied ID, storage revision, fingerprint, marker, or compatibility
|
|
151
|
+
entity. The runnable configuration is in the
|
|
152
|
+
[Message Board deployment](https://github.com/SpineEventEngine/spine-ts/blob/master/examples/message-board/app/src/deployment-config.ts).
|
|
153
|
+
|
|
154
|
+
When a user queries `board == BoardId("board-7")`, Spine converts that generated
|
|
155
|
+
`BoardId` with the same stringifier and sends a Datastore property filter for
|
|
156
|
+
`board == '{"value":"board-7"}'`. Datastore uses the declared property and its
|
|
157
|
+
indexes to find matching keys; Spine decodes the authoritative `bytes` payload
|
|
158
|
+
for each result.
|
|
159
|
+
|
|
160
|
+
## 🔐 Configure credentials and indexes
|
|
161
|
+
|
|
162
|
+
Credential selection belongs to the official Google client. For example, an
|
|
163
|
+
application can use Application Default Credentials or pass `credentials` or
|
|
164
|
+
`keyFilename` in its Datastore options. Keep credentials out of source code and
|
|
165
|
+
logs.
|
|
166
|
+
|
|
167
|
+
Queries that combine equality filters and ordering can require composite
|
|
168
|
+
indexes. Kind names depend on the application's Proto types and layout choices,
|
|
169
|
+
so the framework cannot ship one universal index file. Keep the required
|
|
170
|
+
indexes in the application and deploy them before serving those queries. For
|
|
171
|
+
example:
|
|
172
|
+
|
|
173
|
+
```yaml
|
|
174
|
+
indexes:
|
|
175
|
+
- kind: spine.examples.orders.OrderView
|
|
176
|
+
properties:
|
|
177
|
+
- name: status
|
|
178
|
+
- name: when_created
|
|
179
|
+
direction: desc
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
```sh
|
|
183
|
+
gcloud datastore indexes create index.yaml
|
|
184
|
+
gcloud datastore indexes list
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
Wait until each required index is ready before serving its query combinations.
|
|
188
|
+
|
|
189
|
+
Retained Entity histories need separate indexes because their kinds and
|
|
190
|
+
ordering differ from current state. For a state type named
|
|
191
|
+
`spine.examples.board.Message`, the usual backward-history index is:
|
|
192
|
+
|
|
193
|
+
```yaml
|
|
194
|
+
indexes:
|
|
195
|
+
- kind: spine.examples.board.Message_EntityRecord
|
|
196
|
+
properties:
|
|
197
|
+
- name: entity_id
|
|
198
|
+
- name: version
|
|
199
|
+
direction: desc
|
|
200
|
+
- name: created
|
|
201
|
+
direction: desc
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
`stateAt()` instead orders `created` and then `version`, both descending.
|
|
205
|
+
Diagnostic event history uses the corresponding
|
|
206
|
+
`<state-source>_Event` grouped kind and the same `entity_id`, `version`, and
|
|
207
|
+
`created` columns. Inspect the concrete query shapes your application serves
|
|
208
|
+
and add the matching Datastore composite indexes; the adapter never creates
|
|
209
|
+
them automatically.
|
|
210
|
+
|
|
211
|
+
## 📏 Understand query limits
|
|
212
|
+
|
|
213
|
+
The adapter has a fixed finite client-side reconciliation budget of `1000`.
|
|
214
|
+
When a query needs more candidates than that budget,
|
|
215
|
+
`DatastoreQueryLimitError` is thrown instead of returning a partial result.
|
|
216
|
+
There is no unlimited scan setting or Datastore-specific cursor API.
|
|
217
|
+
Provider filters and ordering are pushed down; public query offsets and mixed
|
|
218
|
+
local/provider limits are reconciled locally within that finite bound.
|
|
219
|
+
|
|
220
|
+
Ordinary `write()` and `writeAll()` calls are independent provider writes. Use
|
|
221
|
+
compare-and-set for an atomic conditional record update, or an Entity commit
|
|
222
|
+
for one atomic current-state/history/event mutation.
|
|
223
|
+
|
|
224
|
+
## 🧪 Verify with a local emulator
|
|
225
|
+
|
|
226
|
+
The optional emulator suite needs an already-running Firestore emulator in
|
|
227
|
+
Datastore mode.
|
|
228
|
+
|
|
229
|
+
```sh
|
|
230
|
+
gcloud emulators firestore start --database-mode=datastore-mode --host-port=127.0.0.1:8081
|
|
231
|
+
DATASTORE_EMULATOR_HOST=127.0.0.1:8081 \
|
|
232
|
+
pnpm --filter @spine-event-engine/storage-datastore test:emulator
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
Use a disposable project for the credential-gated cloud smoke test. Neither
|
|
236
|
+
test substitutes for production index, quota, or consistency planning.
|
|
237
|
+
|
|
238
|
+
## ⚠️ Before production
|
|
239
|
+
|
|
240
|
+
Deploy every composite index used by the application, select quotas and retry
|
|
241
|
+
policy for its workload, and keep Google credentials outside source control.
|
|
242
|
+
This adapter does not provide an unlimited scan mode or a Datastore-specific
|
|
243
|
+
cursor API.
|
|
244
|
+
|
|
245
|
+
Before starting this corrected layout in an existing project, inventory every
|
|
246
|
+
native namespace and kind:
|
|
247
|
+
|
|
248
|
+
```sh
|
|
249
|
+
pnpm --dir packages/storage-datastore inventory:legacy -- --project my-project
|
|
250
|
+
```
|
|
251
|
+
|
|
252
|
+
The command exits nonzero if discovery fails or it finds an old `_scope`
|
|
253
|
+
property or scope-derived key name. Do not start the corrected runtime until it
|
|
254
|
+
passes. Old rows are invisible to direct keys; move them offline, and stop if
|
|
255
|
+
rows from old Bounded Context partitions would become the same namespace,
|
|
256
|
+
kind, and key. The runtime never chooses a winner.
|
|
257
|
+
|
|
258
|
+
## 🔗 Learn more
|
|
259
|
+
|
|
260
|
+
- [Storage API](https://github.com/SpineEventEngine/spine-ts/blob/master/packages/storage/README.md)
|
|
261
|
+
- [Datastore guide](https://github.com/SpineEventEngine/spine-ts/blob/master/docs/USER_GUIDE.md#6-persist-application-data)
|
|
262
|
+
- [Reference for coding agents](REFERENCE.md)
|
package/REFERENCE.md
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
# @spine-event-engine/storage-datastore reference
|
|
2
|
+
|
|
3
|
+
This reference is for agents configuring the Google Cloud Datastore adapter.
|
|
4
|
+
|
|
5
|
+
## Public entry point
|
|
6
|
+
|
|
7
|
+
Import `DatastoreStorageFactory`, `DatastoreStorageFactoryBuilder`,
|
|
8
|
+
`RecordLayout`, `CreateRecordStorage`, `CreateEntityStorage`,
|
|
9
|
+
`NamespaceConverter`, `DefaultNamespaceConverter`, `DatastoreIdColumn`,
|
|
10
|
+
`DatastoreColumnMapping`, and `DatastoreQueryLimitError` from
|
|
11
|
+
`@spine-event-engine/storage-datastore`.
|
|
12
|
+
Create a factory with `DatastoreStorageFactory.newBuilder().setClient(client)`.
|
|
13
|
+
The adapter never closes that client.
|
|
14
|
+
|
|
15
|
+
The builder is mutable until `build()` and the built factory receives a
|
|
16
|
+
snapshot. The last registration for the same identity wins. Record resolution
|
|
17
|
+
uses this precedence:
|
|
18
|
+
|
|
19
|
+
1. an exact source-plus-record creator;
|
|
20
|
+
2. a record-only creator;
|
|
21
|
+
3. an exact source-plus-record layout;
|
|
22
|
+
4. a record-only layout; and
|
|
23
|
+
5. the default kind.
|
|
24
|
+
|
|
25
|
+
`setNamespaceConverter(...)` replaces complete `TenantId`/native namespace
|
|
26
|
+
conversion. `setStringifierRegistry(...)` supplies reversible mappings for
|
|
27
|
+
message-valued IDs and ordinary message columns. The stringifier registry is
|
|
28
|
+
snapshotted by `build()`; a custom namespace converter is retained and must
|
|
29
|
+
remain behaviorally immutable for the factory lifetime. `organizeRecords(...)`
|
|
30
|
+
changes a kind. `useRecordStorage(...)` replaces a record-family provider.
|
|
31
|
+
`Identifiers` select the key representation, and the same configured
|
|
32
|
+
stringifier maps a message-valued ID or declared column on write, direct lookup,
|
|
33
|
+
filter, ordering continuation, and read. `RecordQuery<I>` statically types IDs
|
|
34
|
+
only. Datastore maps filter values through its record column mapping at runtime;
|
|
35
|
+
its sort-property mapping is the requested field name. Callers do not construct
|
|
36
|
+
a provider string or infer shared filter/sort-name validation.
|
|
37
|
+
`useEntityStorage(...)` replaces the complete coherent Entity handle, including
|
|
38
|
+
its commit capability. Custom providers receive the storage context, record or
|
|
39
|
+
Entity contract, and caller-supplied client, but not the built-in converter or
|
|
40
|
+
stringifier registry; they must provide equivalent tenant, ID, and column mapping. The
|
|
41
|
+
built-in Entity provider does not mix custom record creators into one transaction.
|
|
42
|
+
|
|
43
|
+
## Physical layout
|
|
44
|
+
|
|
45
|
+
Every record family uses the source Proto full name as its default kind. A
|
|
46
|
+
grouped family uses `<group>_<record-simple-name>`. A layout registration can
|
|
47
|
+
replace only the kind. Rows contain unindexed Protobuf `bytes` and the declared
|
|
48
|
+
indexed columns. Current Entity rows declare `archived`, `deleted`, and Entity
|
|
49
|
+
`version` in addition to model `(column)` fields; Entity `version` is not a
|
|
50
|
+
provider revision. No scope, ID copy, storage revision, schema fingerprint,
|
|
51
|
+
marker, or compatibility entity is stored. The Bounded Context name is
|
|
52
|
+
diagnostic only.
|
|
53
|
+
|
|
54
|
+
Single tenancy preserves the caller client's configured/default namespace.
|
|
55
|
+
Multitenancy converts the complete generated `TenantId` to a native namespace.
|
|
56
|
+
The default converter supports the reversible JVM `D<domain>` and `V<value>`
|
|
57
|
+
forms. It rejects email tenants because JVM's `E` conversion replaces `@` with
|
|
58
|
+
`-at-` and is not injective. Email tenants require one reversible custom
|
|
59
|
+
converter installed in both TS and JVM. Catalog admission rejects empty,
|
|
60
|
+
non-round-tripping, or colliding converter results before provider work. The
|
|
61
|
+
key contains only the resolved kind and mapped record ID. Message IDs use
|
|
62
|
+
compact Proto JSON or their
|
|
63
|
+
registered custom stringifier; `string`, `int32`, and `int64` IDs use their
|
|
64
|
+
direct text form as the key name. `bytes` and `__key__` are reserved. Blank
|
|
65
|
+
kinds, kinds or key names over 1,500 UTF-8 bytes, unsupported values, non-finite
|
|
66
|
+
numbers, and integers outside their declared/provider range fail before an
|
|
67
|
+
RPC. Stored `bytes` remain authoritative; declared properties are rematerialized
|
|
68
|
+
after decoding.
|
|
69
|
+
|
|
70
|
+
`keep()` bridges native `__namespace__` metadata lag without storing a tenant
|
|
71
|
+
record. Its early-admission cache retains at most 1,000 tenants for at most 60
|
|
72
|
+
seconds and removes an entry sooner when native metadata becomes visible. A
|
|
73
|
+
full cache fails closed; applications can retry after metadata appears or an
|
|
74
|
+
entry expires.
|
|
75
|
+
|
|
76
|
+
Declared string and boolean columns use native values. Integer columns use
|
|
77
|
+
Datastore int values, float/double columns use Datastore double values, bytes
|
|
78
|
+
use blobs, enums use their numeric value, Timestamp uses the native timestamp,
|
|
79
|
+
Version uses its number, and ordinary messages use the same compact Proto JSON
|
|
80
|
+
or custom stringifier as query operands. Null is stored as native null. The
|
|
81
|
+
application type registry is required when default Proto JSON expands `Any`.
|
|
82
|
+
|
|
83
|
+
Entity current records use `EntityRecord`; enabled state history uses grouped
|
|
84
|
+
`EntityStateKey`/`EntityRecord`; diagnostic history uses grouped
|
|
85
|
+
`EventId`/`Event`; Event Store is a separate ungrouped Event family. Disabled
|
|
86
|
+
histories allocate no record handle or Datastore row.
|
|
87
|
+
|
|
88
|
+
## Queries and commits
|
|
89
|
+
|
|
90
|
+
Queries run inside the selected native namespace. Normalized plans admit only
|
|
91
|
+
IDs (up to Datastore's legal key filter bound), equality, one inequality column,
|
|
92
|
+
flat conjunction, inequality-compatible ordering, limit, and mask. Nested or
|
|
93
|
+
disjunctive predicates, oversized key sets, and illegal inequality/order shapes
|
|
94
|
+
reject before provider access; they never trigger an unfiltered reconciliation
|
|
95
|
+
read.
|
|
96
|
+
`writeAll()` uses batches of at most 500 mutations and is not atomic across
|
|
97
|
+
batches.
|
|
98
|
+
|
|
99
|
+
Provider-legal ID predicates, declared-property comparisons, and ordering are
|
|
100
|
+
pushed only when Datastore can execute the whole selected conjunction. Runtime
|
|
101
|
+
descriptor/column validation happens before that decision. An omitted query
|
|
102
|
+
budget uses the shared 10,000-record materialization bound, but Datastore's
|
|
103
|
+
explicit value cannot exceed 10,000. Datastore's accepted/provider ceiling is
|
|
104
|
+
1,000 rows. It may read one overflow-lookahead row
|
|
105
|
+
(1,001 raw rows) to reject an oversized scan rather than materializing more
|
|
106
|
+
rows. There is no public Datastore cursor API.
|
|
107
|
+
|
|
108
|
+
This is deliberately the overlap, not MySQL parity: nested or disjunctive
|
|
109
|
+
predicates and provider-illegal inequality/order shapes reject before provider
|
|
110
|
+
access. Normalized plans have no offset; `RecordQuery.offset` is separate.
|
|
111
|
+
|
|
112
|
+
The internal Entity commit reads current and immutable keys then applies current,
|
|
113
|
+
enabled histories, and delivery events in one Datastore transaction. It rejects
|
|
114
|
+
more than 25 entity groups, 500 mutations, or its conservative transaction-size
|
|
115
|
+
limit before opening the transaction. Current mismatch returns `conflict`.
|
|
116
|
+
An already-applied identical retry returns `committed`; divergent immutable
|
|
117
|
+
content fails. Only ABORTED provider failures retry, for at most three attempts.
|
|
118
|
+
|
|
119
|
+
History reads use stable finite keyset pages. State history provides backward,
|
|
120
|
+
state-at-time, trim, and truncate behavior; event history provides backward and
|
|
121
|
+
truncate behavior. Timestamp comparisons include seconds and nanoseconds.
|
|
122
|
+
Long maintenance can commit several bounded chunks, so a later failure leaves
|
|
123
|
+
earlier chunks durable and the caller retries the same idempotent operation.
|
|
124
|
+
|
|
125
|
+
The factory has one tenant catalog. It reads native `__namespace__` metadata,
|
|
126
|
+
converts only namespaces recognized by its `NamespaceConverter`, and keeps an early
|
|
127
|
+
in-memory cache for newly admitted tenants. `keep()` stores no `TenantId` row or
|
|
128
|
+
other discovery record; Datastore exposes native metadata after an application
|
|
129
|
+
entity is written in that namespace.
|
|
130
|
+
|
|
131
|
+
Before deployment to an existing project, run `pnpm --dir
|
|
132
|
+
packages/storage-datastore inventory:legacy -- --project <project-id>`. It
|
|
133
|
+
enumerates native namespaces and kinds and fails closed on discovery errors,
|
|
134
|
+
an old `_scope` property, or a scope-derived key name. Passing the inventory is
|
|
135
|
+
a startup prerequisite; the application performs migration offline, with no
|
|
136
|
+
dual-layout reads or automatic conflict winner.
|
|
137
|
+
|
|
138
|
+
## Operations and errors
|
|
139
|
+
|
|
140
|
+
Malformed payloads fail with a redacted decoding error. Physical Datastore
|
|
141
|
+
errors are surfaced without payload data. The emulator suite is required for
|
|
142
|
+
provider acceptance; cloud smoke remains credential-gated.
|
|
143
|
+
|
|
144
|
+
`build()` and storage-handle creation issue no request. Closing the factory is
|
|
145
|
+
idempotent and prevents creation of another handle without closing the
|
|
146
|
+
caller-supplied client or already-created handles. Each record, Entity, and commit
|
|
147
|
+
handle has an independent idempotent lifecycle and rejects work after it is closed.
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { StringifierRegistry } from "@spine-event-engine/core";
|
|
2
|
+
import type { ColumnMapping, ColumnTypeMapping, RecordColumnType } from "@spine-event-engine/storage";
|
|
3
|
+
/**
|
|
4
|
+
* Converts typed Protobuf column values to Spine JVM Datastore values.
|
|
5
|
+
*/
|
|
6
|
+
export declare class DatastoreColumnMapping implements ColumnMapping<unknown> {
|
|
7
|
+
#private;
|
|
8
|
+
/**
|
|
9
|
+
* Creates a JVM-compatible Datastore column mapping.
|
|
10
|
+
*
|
|
11
|
+
* @param stringifiers The schema-bound message stringifiers.
|
|
12
|
+
*/
|
|
13
|
+
constructor(stringifiers?: StringifierRegistry);
|
|
14
|
+
/**
|
|
15
|
+
* Returns the conversion for one generated column type.
|
|
16
|
+
*
|
|
17
|
+
* @param type The generated Protobuf column type.
|
|
18
|
+
* @returns The JVM-compatible Datastore conversion.
|
|
19
|
+
*/
|
|
20
|
+
of<V>(type: RecordColumnType<V>): ColumnTypeMapping<V, unknown>;
|
|
21
|
+
/**
|
|
22
|
+
* Returns native Datastore null.
|
|
23
|
+
*
|
|
24
|
+
* @returns The null conversion.
|
|
25
|
+
*/
|
|
26
|
+
ofNull(): ColumnTypeMapping<null, unknown>;
|
|
27
|
+
}
|
|
28
|
+
//# sourceMappingURL=column-mapping.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"column-mapping.d.ts","sourceRoot":"","sources":["../../src/datastore/column-mapping.ts"],"names":[],"mappings":"AAiBA,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAC/D,OAAO,KAAK,EACV,aAAa,EACb,iBAAiB,EACjB,gBAAgB,EACjB,MAAM,6BAA6B,CAAC;AAKrC;;GAEG;AACH,qBAAa,sBAAuB,YAAW,aAAa,CAAC,OAAO,CAAC;;IAGnE;;;;OAIG;gBACS,YAAY,GAAE,mBAA+C;IAIzE;;;;;OAKG;IACH,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,gBAAgB,CAAC,CAAC,CAAC,GAAG,iBAAiB,CAAC,CAAC,EAAE,OAAO,CAAC;IAkB/D;;;;OAIG;IACH,MAAM,IAAI,iBAAiB,CAAC,IAAI,EAAE,OAAO,CAAC;CAG3C"}
|