@spine-event-engine/storage-datastore 2.0.0-snapshot.21 → 2.0.0-snapshot.23

package/README.md

@@ -11,6 +11,11 @@ record schemas, and a configured Google Datastore client.
 For exact query limits, entity-storage behavior, and failure handling, see
 [REFERENCE documentation for agents](REFERENCE.md).
 
+For an application using Agents, this provider keeps the Agent's accepted
+work and recorded interactions across restarts. Deploy the supplied `index.yaml`
+for Agent history and execution queries; see the [coding-agent reference](REFERENCE.md)
+for provider requirements.
+
 ## 💡 Why use it?
 
 - ✅ Stores Spine records in Google Cloud Datastore or Firestore in Datastore mode.

package/REFERENCE.md

@@ -1,6 +1,6 @@
 # @spine-event-engine/storage-datastore reference
 
-This reference is for agents configuring the Google Cloud Datastore adapter.
+This reference is for coding agents configuring the Google Cloud Datastore adapter.
 
 ## Public entry point
 
@@ -124,6 +124,20 @@ truncate behavior. Timestamp comparisons include seconds and nanoseconds.
 Long maintenance can commit several bounded chunks, so a later failure leaves
 earlier chunks durable and the caller retries the same idempotent operation.
 
+The provider-only Agent history handle uses the fixed `spine_agent_history`
+kind in each complete tenant namespace. Each row retains the original complete
+`AgentHistoryEntry` and full state type and Agent key. SHA-256 digests provide
+bounded physical IDs and indexed scope/conversation keys; every decoded row is
+checked against the complete requested scope and conversation. A digest
+collision fails the read. `index.yaml` declares three complete composite
+indexes for scope plus full order, scope plus category plus full order, and
+scope plus conversation digest plus full order. The order value encodes full
+Timestamp seconds/nanoseconds, category, and unsigned UTF-8 record ID. Values
+longer than Datastore's 1,500-byte indexed-string limit reject before append;
+original scope strings remain unindexed in the payload. Reads use native pages
+of at most 128 rows and sum serialized entry bytes. A fetched chunk can be
+larger than the returned byte budget.
+
 The factory has one tenant catalog. It reads native `__namespace__` metadata,
 converts only namespaces recognized by its `NamespaceConverter`, and keeps an early
 in-memory cache for newly admitted tenants. `keep()` stores no `TenantId` row or
@@ -137,6 +151,32 @@ an old `_scope` property, or a scope-derived key name. Passing the inventory is
 a startup prerequisite; the application performs migration offline, with no
 dual-layout reads or automatic conflict winner.
 
+## Agent execution
+
+Agent execution uses `spine_agent_execution` and `spine_agent_execution_head` in
+the selected tenant namespace. Deploy the execution indexes in this package's
+`index.yaml` together with its Agent history indexes. Pending queries select one
+entry per instance by its complete eligibility-time key.
+
+The database must support the adapter's non-ancestor query inside a transaction.
+The `OPTIMISTIC_WITH_ENTITY_GROUPS` concurrency mode requires ancestor queries
+and is therefore incompatible with Agent execution in this adapter. See
+[Datastore transaction modes](https://docs.cloud.google.com/datastore/docs/concepts/transactions).
+Admission probes that query and the indexed pending query before saving work.
+A database mode that forbids it is rejected; the adapter does not fall back to
+a scan or a query outside the transaction. The local Agent execution conformance suite uses the Firestore emulator in
+Datastore mode, distributed with Google Cloud CLI 578.0.0. This tests the
+transactional query capability; it does not validate a production project's
+configuration.
+
+Conditional completion reads the current Entity Version and pending work before
+queuing writes. State, mandatory history, preferences, execution completion and
+outgoing signals are saved in the same transaction. Datastore entity, index and
+transaction limits still apply: execution, per-instance and Agent history
+payloads are limited to 1,000,000 encoded bytes each, and the adapter limits a staged transaction to
+9 MiB. Response reservations must account for the complete stored records before
+inference. No transaction remains open during a model call.
+
 ## Operations and errors
 
 Malformed payloads fail with a redacted decoding error. Physical Datastore

package/dist/datastore/agent-execution.d.ts

@@ -0,0 +1,337 @@
+import { type Message } from "@bufbuild/protobuf";
+import type { Timestamp } from "@bufbuild/protobuf/wkt";
+import { type Event, type EventId } from "@spine-event-engine/proto";
+import { type AgentAcceptedInvocation, type AgentExecutionHead, type AgentExecutionRecord, type AgentInvocationKey, type AgentSignalKey } from "@spine-event-engine/proto/generated/spine/server/agent/execution_record_pb.js";
+import type { AgentHistoryRecord } from "@spine-event-engine/proto/generated/spine/server/agent/history_record_pb.js";
+import type { EntityRecord } from "@spine-event-engine/proto/generated/spine/server/entity/entity_pb.js";
+import { type AgentExecutionStorage, type AgentExecutionCapacity, type AgentExecutionStorageInput, type AgentExecutionClaim, type AgentExecutionUpdate, type AgentExecutionComplete, type AgentPendingRead, type AgentPendingPage } from "@spine-event-engine/storage/provider";
+import { DatastoreRecordStorage } from "./record-storage.js";
+/**
+ * Existing native record families participating in one Agent transaction.
+ *
+ * @typeParam I Typed entity identifier.
+ */
+export interface DatastoreExecutionRows<I> {
+    /**
+     * Mutable execution evidence.
+     */
+    readonly invocation: DatastoreRecordStorage<string, AgentExecutionRecord>;
+    /**
+     * Per-instance discovery and claim row.
+     */
+    readonly head: DatastoreRecordStorage<string, AgentExecutionHead>;
+    /**
+     * Immutable conversation, System and domain history.
+     */
+    readonly history: DatastoreRecordStorage<string, AgentHistoryRecord>;
+    /**
+     * Current Entity state and Version.
+     */
+    readonly current: DatastoreRecordStorage<I, EntityRecord>;
+    /**
+     * Optional retained state-history records.
+     */
+    readonly states?: DatastoreRecordStorage<unknown, EntityRecord>;
+    /**
+     * Optional retained diagnostic Event records.
+     */
+    readonly diagnostics?: DatastoreRecordStorage<unknown, Event>;
+    /**
+     * Canonical outgoing Event envelopes.
+     */
+    readonly events: DatastoreRecordStorage<EventId, Event>;
+}
+/**
+ * Persists Agent execution evidence through native namespace-scoped transactions.
+ *
+ * @typeParam I Typed Agent identifier.
+ * @typeParam S Generated Agent state.
+ */
+export declare class DatastoreAgentExecution<I, S extends Message> implements AgentExecutionStorage<I, S> {
+    #private;
+    private readonly input;
+    private readonly rows;
+    /**
+     * Entity and staged-transaction limits for complete internal Protobuf wrappers.
+     */
+    readonly capacity: Required<AgentExecutionCapacity>;
+    /**
+     * Binds one complete tenant and generated state type to native record kinds.
+     *
+     * @param input Requested fenced execution change.
+     * @param rows Provider record handles bound to the tenant.
+     */
+    constructor(input: AgentExecutionStorageInput<I, S>, rows: DatastoreExecutionRows<I>);
+    /**
+     * Persists source facts once after proving transaction-query capability.
+     *
+     * @param accepted Original accepted signal and selected bindings.
+     * @returns Persisted invocation with the original accepted facts.
+     */
+    admit(accepted: AgentAcceptedInvocation): Promise<AgentExecutionRecord>;
+    /**
+     * Reads a bounded native head page before a fixed provider-time cutoff.
+     *
+     * @param request Bounded pending-page request.
+     * @returns Bounded eligible work and continuation cursor.
+     */
+    pending(request: AgentPendingRead): Promise<AgentPendingPage>;
+    /**
+     * Reads no more than count plus one complete native head keys.
+     *
+     * @param asOf Fixed eligibility cutoff for the page.
+     * @param request Bounded pending-page request.
+     * @returns Bounded native page of eligible per-instance heads.
+     */
+    private pendingHeads;
+    /**
+     * Reads one complete retained invocation after reconstruction.
+     *
+     * @param key Original Agent invocation identity.
+     * @returns Complete stored invocation when present.
+     */
+    read(key: AgentInvocationKey): Promise<AgentExecutionRecord | undefined>;
+    /**
+     * Acquires the earliest same-instance candidate with a fresh transaction clock.
+     *
+     * @param expiresAt Requested future lease expiry.
+     * @param key Original Agent invocation identity.
+     * @param token Exclusive claim token.
+     * @returns Claimed work and preferences when eligible.
+     */
+    claim(key: AgentInvocationKey, token: string, expiresAt: Timestamp): Promise<AgentExecutionClaim | undefined>;
+    /**
+     * Checks the current head and invocation before writing one exclusive lease.
+     *
+     * @param expiresAt Requested future lease expiry.
+     * @param key Original Agent invocation identity.
+     * @param token Exclusive claim token.
+     * @param tx Native transaction that fences the change.
+     * @returns Claimed record and preferences when eligible.
+     */
+    private claimLocked;
+    /**
+     * Checks the current per-instance head and earliest unresolved source.
+     *
+     * @param tx Native transaction that fences the claim.
+     * @param head Per-instance pending and lease record.
+     * @param key Original Agent invocation identity.
+     * @param now Fresh time read inside the transaction.
+     * @returns Whether this invocation may receive the next lease.
+     */
+    private eligibleClaim;
+    /**
+     * Updates the same live token inside a fresh native transaction.
+     *
+     * @param expiresAt Requested future lease expiry.
+     * @param key Original Agent invocation identity.
+     * @param token Exclusive claim token.
+     * @returns Whether the current lease was extended.
+     */
+    renew(key: AgentInvocationKey, token: string, expiresAt: Timestamp): Promise<boolean>;
+    /**
+     * Persists evidence and immutable history inside one native transaction.
+     *
+     * @param input Requested fenced execution change.
+     * @returns Completion after fenced evidence is persisted.
+     */
+    update(input: AgentExecutionUpdate): Promise<void>;
+    /**
+     * Commits Entity, history, preferences and outgoing IDs together.
+     *
+     * @param input Requested fenced execution change.
+     * @returns Completion after conditional Entity and evidence commit.
+     */
+    complete(input: AgentExecutionComplete<I, S>): Promise<void>;
+    /**
+     * Marks original output IDs delivered without rerunning completed handlers.
+     *
+     * @param expectedRecordBytes Exact prior execution record bytes.
+     * @param key Original Agent invocation identity.
+     * @param signals Original outgoing signal identities.
+     * @param token Exclusive claim token.
+     * @returns Completion after original output IDs are marked delivered.
+     */
+    markDelivered(key: AgentInvocationKey, token: string, expectedRecordBytes: Uint8Array, signals: readonly AgentSignalKey[]): Promise<void>;
+    /**
+     * Closes provider handles without deleting accepted work.
+     */
+    close(): void;
+    /**
+     * Rejects use after closure and probes native transaction-query support once.
+     *
+     * @returns Completion after provider capabilities are checked.
+     */
+    private ready;
+    /**
+     * Checks flat scoped successor and head queries before accepting a signal.
+     *
+     * @returns Completion after native query support is verified.
+     */
+    private checkCapability;
+    /**
+     * Checks that native indexed head pagination is available.
+     *
+     * @returns Completion after the indexed query succeeds.
+     */
+    private checkHeadCapability;
+    /**
+     * Executes one namespace-scoped transaction and retries native contention.
+     *
+     * @param work Operation executed inside the transaction.
+     * @returns Callback result after the native transaction commits.
+     * @typeParam T Result returned by the transaction callback.
+     */
+    private transaction;
+    /**
+     * Reads one head by complete scope inside the active transaction.
+     *
+     * @param key Original Agent invocation identity.
+     * @param tx Native transaction that fences the change.
+     * @returns Per-instance head when present.
+     */
+    private head;
+    /**
+     * Reads one invocation by original source and complete scope.
+     *
+     * @param key Original Agent invocation identity.
+     * @param tx Native transaction that fences the change.
+     * @returns Stored invocation when present.
+     */
+    private invocation;
+    /**
+     * Reads a materialized key and decodes exact original Protobuf bytes.
+     *
+     * @param id Complete persisted record identity.
+     * @param storage Native record storage for this row.
+     * @param tx Native transaction that fences the change.
+     * @returns Stored typed row when present.
+     * @typeParam Id Typed record identifier.
+     * @typeParam R Typed persisted record.
+     */
+    private readRow;
+    /**
+     * Queues one validated native row for the current short transaction.
+     *
+     * @param record Persisted Agent execution record.
+     * @param storage Native record storage for this row.
+     * @param tx Native transaction that fences the change.
+     * @typeParam Id Typed record identifier.
+     * @typeParam R Typed persisted record.
+     */
+    private save;
+    /**
+     * Checks fresh Time, exact token, and the persisted claim image.
+     *
+     * @param key Original Agent invocation identity.
+     * @param token Exclusive claim token.
+     * @param tx Native transaction that fences the change.
+     * @returns Current record and head if the token remains valid.
+     */
+    private claimed;
+    /**
+     * Checks exact prior bytes and unchanged immutable source facts.
+     *
+     * @param checkNext Whether to validate the proposed next record.
+     * @param input Requested fenced execution change.
+     * @param tx Native transaction that fences the change.
+     * @returns Current fenced record and head.
+     */
+    private expected;
+    /**
+     * Updates the pending head for one earlier accepted source without resetting existing eligibility.
+     *
+     * @param head Per-instance pending and lease record.
+     * @param record Persisted Agent execution record.
+     */
+    private offerCandidate;
+    /**
+     * Reads up to two same-instance rows per unresolved status inside the transaction.
+     *
+     * @param exclude Invocation omitted while selecting the successor.
+     * @param key Original Agent invocation identity.
+     * @param tx Native transaction that fences the change.
+     * @returns Earliest unresolved invocation when present.
+     */
+    private successor;
+    /**
+     * Reads the first two rows of one unresolved status under the instance scope.
+     *
+     * @param tx Native transaction that fences successor selection.
+     * @param scope Generated state type and canonical Agent key.
+     * @param status Persisted unresolved invocation status.
+     * @returns Native rows in complete Inbox order.
+     */
+    private successorsForStatus;
+    /**
+     * Updates the next original Inbox source after terminal work.
+     *
+     * @param head Per-instance pending and lease record.
+     * @param key Original Agent invocation identity.
+     * @param order Original Inbox order.
+     * @param tx Native transaction that fences the change.
+     * @returns Completion after updating the next pending head.
+     */
+    private advance;
+    /**
+     * Reads immutable history keys before queuing any transaction writes.
+     *
+     * @param entries Immutable history entries to append.
+     * @param key Original Agent invocation identity.
+     * @param tx Native transaction that fences the change.
+     * @returns Completion after immutable history is appended.
+     */
+    private appendHistory;
+    /**
+     * Persists one missing immutable row or rejects divergent existing bytes.
+     *
+     * @param record Persisted Agent execution record.
+     * @param storage Native record storage for this row.
+     * @param tx Native transaction that fences the change.
+     * @returns Completion after immutable row insertion or exact-byte verification.
+     * @typeParam Id Typed record identifier.
+     * @typeParam R Typed persisted record.
+     */
+    private immutable;
+    /**
+     * Checks initial Version and stages Entity records in this transaction.
+     *
+     * @param expected Expected current Entity Version.
+     * @param input Requested fenced execution change.
+     * @param tx Native transaction that fences the change.
+     * @returns Completion after conditional Entity state and Version write.
+     */
+    private completeEntity;
+    /**
+     * Rejects cross-Entity completion and disabled framework histories.
+     *
+     * @param commit Conditional Entity state and Version write.
+     * @param id Complete persisted record identity.
+     */
+    private validateEntityCommit;
+    /**
+     * Checks exactly one Version advance or an unchanged state.
+     *
+     * @param input Requested fenced execution change.
+     * @param prior Previously persisted execution record.
+     * @returns Expected initial Entity Version.
+     */
+    private validateCompletion;
+    /**
+     * Checks native indexed metadata and per-entity payload limits.
+     *
+     * @param record Persisted Agent execution record.
+     */
+    private validateRecord;
+    /**
+     * Checks complete head discovery key and per-entity payload limits.
+     *
+     * @param head Per-instance pending and lease record.
+     */
+    private validateHead;
+    /**
+     * Rejects provider calls after closure.
+     */
+    private requireOpen;
+}
+//# sourceMappingURL=agent-execution.d.ts.map

package/dist/datastore/agent-execution.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-execution.d.ts","sourceRoot":"","sources":["../../src/datastore/agent-execution.ts"],"names":[],"mappings":"AAcA,OAAO,EAAoB,KAAK,OAAO,EAAE,MAAM,oBAAoB,CAAC;AACpE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAExD,OAAO,EAAE,KAAK,KAAK,EAAE,KAAK,OAAO,EAAE,MAAM,2BAA2B,CAAC;AAErE,OAAO,EAIL,KAAK,uBAAuB,EAC5B,KAAK,kBAAkB,EACvB,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,cAAc,EACpB,MAAM,+EAA+E,CAAC;AACvF,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,6EAA6E,CAAC;AACtH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sEAAsE,CAAC;AACzG,OAAO,EAOL,KAAK,qBAAqB,EAC1B,KAAK,sBAAsB,EAC3B,KAAK,0BAA0B,EAC/B,KAAK,mBAAmB,EACxB,KAAK,oBAAoB,EACzB,KAAK,sBAAsB,EAC3B,KAAK,gBAAgB,EACrB,KAAK,gBAAgB,EACtB,MAAM,sCAAsC,CAAC;AAI9C,OAAO,EACL,sBAAsB,EAGvB,MAAM,qBAAqB,CAAC;AAE7B;;;;GAIG;AACH,MAAM,WAAW,sBAAsB,CAAC,CAAC;IACvC;;OAEG;IACH,QAAQ,CAAC,UAAU,EAAE,sBAAsB,CAAC,MAAM,EAAE,oBAAoB,CAAC,CAAC;IAE1E;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,sBAAsB,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IAElE;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,sBAAsB,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IAErE;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,sBAAsB,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IAE1D;;OAEG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,sBAAsB,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IAEhE;;OAEG;IACH,QAAQ,CAAC,WAAW,CAAC,EAAE,sBAAsB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IAE9D;;OAEG;IACH,QAAQ,CAAC,MAAM,EAAE,sBAAsB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;CACzD;AAOD;;;;;GAKG;AACH,qBAAa,uBAAuB,CAAC,CAAC,EAAE,CAAC,SAAS,OAAO,CAAE,YAAW,qBAAqB,CAAC,CAAC,EAAE,CAAC,CAAC;;IAwB7F,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAxBvB;;OAEG;IACH,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,sBAAsB,CAAC,CAKhD;IAQH;;;;;OAKG;gBAEgB,KAAK,EAAE,0BAA0B,CAAC,CAAC,EAAE,CAAC,CAAC,EACvC,IAAI,EAAE,sBAAsB,CAAC,CAAC,CAAC;IAGlD;;;;;OAKG;IACG,KAAK,CAAC,QAAQ,EAAE,uBAAuB,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAwB7E;;;;;OAKG;IACG,OAAO,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA0BnE;;;;;;OAMG;YACW,YAAY;IAoC1B;;;;;OAKG;IACG,IAAI,CAAC,GAAG,EAAE,kBAAkB,GAAG,OAAO,CAAC,oBAAoB,GAAG,SAAS,CAAC;IAS9E;;;;;;;OAOG;IACG,KAAK,CACT,GAAG,EAAE,kBAAkB,EACvB,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,SAAS,GACnB,OAAO,CAAC,mBAAmB,GAAG,SAAS,CAAC;IAK3C;;;;;;;;OAQG;YACW,WAAW;IA8BzB;;;;;;;;OAQG;YACW,aAAa;IAsB3B;;;;;;;OAOG;IACG,KAAK,CAAC,GAAG,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAkB3F;;;;;OAKG;IACG,MAAM,CAAC,KAAK,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBxD;;;;;OAKG;IACG,QAAQ,CAAC,KAAK,EAAE,sBAAsB,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAwBlE;;;;;;;;OAQG;IACG,aAAa,CACjB,GAAG,EAAE,kBAAkB,EACvB,KAAK,EAAE,MAAM,EACb,mBAAmB,EAAE,UAAU,EAC/B,OAAO,EAAE,SAAS,cAAc,EAAE,GACjC,OAAO,CAAC,IAAI,CAAC;IAoBhB;;OAEG;IACH,KAAK,IAAI,IAAI;IAYb;;;;OAIG;IACH,OAAO,CAAC,KAAK;IAQb;;;;OAIG;YACW,eAAe;IAsB7B;;;;OAIG;YACW,mBAAmB;IAmBjC;;;;;;OAMG;YACW,WAAW;IAwBzB;;;;;;OAMG;YACW,IAAI;IAmBlB;;;;;;OAMG;YACW,UAAU;IASxB;;;;;;;;;OASG;YACW,OAAO;IAWrB;;;;;;;;OAQG;IACH,OAAO,CAAC,IAAI;IAaZ;;;;;;;OAOG;YACW,OAAO;IAgBrB;;;;;;;OAOG;YACW,QAAQ;IAetB;;;;;OAKG;IACH,OAAO,CAAC,cAAc;IAKtB;;;;;;;OAOG;YACW,SAAS;IA2BvB;;;;;;;OAOG;IACH,OAAO,CAAC,mBAAmB;IAmB3B;;;;;;;;OAQG;YACW,OAAO;IAWrB;;;;;;;OAOG;YACW,aAAa;IAsB3B;;;;;;;;;OASG;YACW,SAAS;IAgBvB;;;;;;;OAOG;YACW,cAAc;IAsB5B;;;;;OAKG;IACH,OAAO,CAAC,oBAAoB;IAe5B;;;;;;OAMG;IACH,OAAO,CAAC,kBAAkB;IA+B1B;;;;OAIG;IACH,OAAO,CAAC,cAAc;IAatB;;;;OAIG;IACH,OAAO,CAAC,YAAY;IAcpB;;OAEG;IACH,OAAO,CAAC,WAAW;CAGpB"}