@segmentstream-cloud/ontology 0.25.1 → 0.26.0

package/dist/runtime/sql-runtime.js

@@ -1,9 +1,11 @@
-import { Effect, Option } from "effect";
+import { Effect, Exit, Option } from "effect";
 import { beginReplicationWrite } from "../replication/transaction.js";
 import { isSqlError } from "effect/unstable/sql/SqlError";
-import { Conflict } from "../errors.js";
+import { ActionRejected, Conflict } from "../errors.js";
 import { toSnakeCase } from "../names.js";
 import { User } from "../user.js";
+import { ACTION_MAX_BATCH_ROWS, BATCH_MAX_BYTES, BATCH_MAX_ROWS, batchInputBytes, executeBatch, planBatch, } from "./batch.js";
+import { logBatchConflict } from "./batch-diagnostics.js";
 import { InvalidQuery, WriteOutsideAction } from "./errors.js";
 import { entityModel, ontologyModel } from "./model.js";
 import { loadRecord, notFound, readHistory, relatedCondition, runQuery, validateQuery, } from "./reads.js";
@@ -23,7 +25,7 @@ function withoutUndefined(value) {
         .map(([key, entry]) => [key, withoutUndefined(entry)]));
 }
 /** A unique violation becomes Conflict, naming the property when the index covers one column. */
-function uniqueConflict(entity, error) {
+function uniqueConflict(entity, error, batch) {
     const cause = (error.reason.cause ?? {});
     const unique = error.reason._tag === "UniqueViolation" || cause.code === "23505";
     if (!unique) {
@@ -31,17 +33,36 @@ function uniqueConflict(entity, error) {
     }
     const columns = typeof cause.detail === "string" ? /^Key \(([^)]*)\)=/.exec(cause.detail)?.[1] : undefined;
     const field = entity.fields.find((candidate) => candidate.column === columns);
+    const fields = typeof columns === "string"
+        ? columns.split(", ").flatMap((column) => {
+            const candidate = entity.fields.find((entry) => entry.column === column);
+            return candidate === undefined ? [] : [candidate.relationship ?? candidate.name];
+        })
+        : [];
+    const details = batch ? { batch: { indices: [], ...(fields.length > 0 ? { fields } : {}) } } : {};
     const label = entity.label.toLowerCase();
-    return Effect.fail(field === undefined
+    const conflict = field === undefined
         ? new Conflict({
             entity: entity.name,
             message: `The ${label} conflicts with an existing one.`,
+            ...details,
         })
         : new Conflict({
             entity: entity.name,
             property: field.name,
             message: `A ${label} with this ${field.name} already exists.`,
-        }));
+            ...details,
+        });
+    if (batch) {
+        // Keep the driver's complete cause available to operators without serializing SQL or values.
+        Object.defineProperty(conflict, "cause", { value: error, enumerable: false });
+        return logBatchConflict(entity, error, {
+            operation: batch.operation,
+            action: batch.scope.action,
+            actionId: batch.scope.actionId,
+        }).pipe(Effect.andThen(Effect.fail(conflict)));
+    }
+    return Effect.fail(conflict);
 }
 /**
  * Platform identity bookkeeping touches only User properties and their history. User is excluded
@@ -91,6 +112,8 @@ export function makeOntologyRuntime(ontology, sql, scope) {
         run: (text, params) => client.unsafe(text, params.values),
     };
     const entityOf = (name) => Effect.sync(() => entityModel(model, name));
+    let attemptedBatchRows = 0;
+    let attributionInitialized = false;
     const write = (entityName, operation) => Effect.gen(function* () {
         const entity = yield* entityOf(entityName);
         if (scope === undefined) {
@@ -108,14 +131,111 @@ export function makeOntologyRuntime(ontology, sql, scope) {
         for (const edit of edits) {
             scope.onEdit?.(edit);
         }
+        if (edits.length > 0 && scope.execution !== undefined) {
+            attributionInitialized = true;
+        }
         return result;
     });
+    const batch = (entityName, operation, input) => Effect.suspend(() => {
+        const startedAt = Date.now();
+        let statements = 0;
+        let distinctReferences = 0;
+        let counts = { created: 0, updated: 0, unchanged: 0 };
+        const run = Effect.gen(function* () {
+            if (scope === undefined) {
+                return yield* Effect.die(new WriteOutsideAction({
+                    entity: entityName,
+                    message: `Writes to ${entityName} run inside an action, which history records.`,
+                }));
+            }
+            const declared = ontology.entities.find((entity) => entity.name === entityName);
+            if (declared?.builtIn === true || declared?.entity.definition.source !== undefined) {
+                return yield* new ActionRejected({
+                    message: "Batch writes require an ordinary stored entity.",
+                    batch: { indices: [] },
+                });
+            }
+            const entity = yield* entityOf(entityName);
+            const length = input.records.length;
+            if (length > 0) {
+                attemptedBatchRows += length;
+            }
+            if (length > BATCH_MAX_ROWS || (length > 0 && attemptedBatchRows > ACTION_MAX_BATCH_ROWS)) {
+                return yield* new ActionRejected({
+                    message: length > BATCH_MAX_ROWS
+                        ? `A batch accepts at most ${BATCH_MAX_ROWS} records.`
+                        : `An action accepts at most ${ACTION_MAX_BATCH_ROWS} attempted batch records.`,
+                    batch: { indices: [] },
+                });
+            }
+            if (batchInputBytes(input) > BATCH_MAX_BYTES) {
+                return yield* new ActionRejected({
+                    message: `A batch accepts at most ${BATCH_MAX_BYTES} UTF-8 JSON bytes.`,
+                    batch: { indices: [] },
+                });
+            }
+            const plan = yield* planBatch(entity, operation, input);
+            distinctReferences = plan.distinctReferences;
+            if (length === 0) {
+                return plan.returning === "summary" ? counts : { ...counts, items: [] };
+            }
+            const enclosingTransaction = yield* Effect.serviceOption(sql.transactionService);
+            if (Option.isNone(enclosingTransaction)) {
+                return yield* Effect.die(new WriteOutsideAction({
+                    entity: entityName,
+                    message: "Batch writes require the enclosing action transaction.",
+                }));
+            }
+            const edits = [];
+            statements += 1; // The SQL client's nested transaction issues one savepoint.
+            const result = yield* sql
+                .withTransaction((scope.replicationInitialized ? Effect.void : beginReplicationWrite(sql)).pipe(Effect.andThen(executeBatch({
+                ...context,
+                scope,
+                edits,
+                attributionInitialized,
+                run: (text, params) => {
+                    statements += 1;
+                    return context.run(text, params);
+                },
+            }, entity, plan)), Effect.onExit((exit) => Effect.sync(() => {
+                if (Exit.isFailure(exit)) {
+                    statements += 1; // A failed nested transaction rolls back to its savepoint.
+                }
+            }))))
+                .pipe(Effect.catchIf(isSqlError, (error) => uniqueConflict(entity, error, { operation, scope })));
+            counts = { created: result.created, updated: result.updated, unchanged: result.unchanged };
+            for (const edit of edits) {
+                scope.onEdit?.(edit);
+            }
+            if (edits.length > 0 && scope.execution !== undefined) {
+                attributionInitialized = true;
+            }
+            return result;
+        });
+        return run.pipe(Effect.onExit((exit) => Effect.logDebug("ontology.action.batch_completed").pipe(Effect.annotateLogs({
+            operation,
+            entity: entityName,
+            action: scope?.action,
+            actionId: scope?.actionId,
+            inputCount: input.records.length,
+            distinctReferenceCount: distinctReferences,
+            changedCount: counts.created + counts.updated,
+            unchangedCount: counts.unchanged,
+            sqlStatements: statements,
+            batchMs: Date.now() - startedAt,
+            outcome: Exit.isSuccess(exit) ? "success" : "failure",
+        }))));
+    });
     const query = (input) => Effect.gen(function* () {
         const entity = yield* entityOf(input.entity);
         const validated = yield* validateQuery(entity, input.query);
         return yield* orDieOnSql(runQuery(context, entity, validated));
     });
     return {
+        createMany: ((input) => batch(input.entity, "create", input)),
+        updateMany: ((input) => batch(input.entity, "update", input)),
+        upsertMany: ((input) => batch(input.entity, "upsert", input)),
         get: (input) => orDieOnSql(Effect.gen(function* () {
             const entity = yield* entityOf(input.entity);
             const record = yield* loadRecord(context, entity, input.id, input.forUpdate === true ? "update" : undefined);

package/dist/runtime.d.ts

@@ -1,5 +1,5 @@
 import { Context, Effect } from "effect";
-import type { Conflict, NotFound } from "./errors.js";
+import type { ActionRejected, Conflict, NotFound } from "./errors.js";
 import type { Where } from "./query.js";
 import type { WorkloadExecution } from "./runtime/workload.js";
 /** Who runs an action: an app member or a service principal, both `user` records. */
@@ -76,6 +76,51 @@ export interface UpsertResult<R = StoredRecord> {
     readonly record: R;
     readonly created: boolean;
 }
+/** Whether a batch returns its records or only outcome counts. Defaults to `records`. */
+export type BatchReturning = "records" | "summary";
+export interface BatchOptions {
+    readonly returning?: BatchReturning;
+}
+/** Outcome counts, which together equal the number of inputs in a successful batch. */
+export interface BatchSummaryResult {
+    readonly created: number;
+    readonly updated: number;
+    readonly unchanged: number;
+}
+/** One result per input, in input order, including records whose values did not change. */
+export interface BatchRecordResult<R = StoredRecord> extends BatchSummaryResult {
+    readonly items: ReadonlyArray<{
+        readonly index: number;
+        readonly status: "created" | "updated" | "unchanged";
+        readonly record: R;
+    }>;
+}
+export type BatchResult<R = StoredRecord> = BatchRecordResult<R> | BatchSummaryResult;
+/** Raw batch inputs use wire entity and field names. */
+export interface CreateManyInput extends BatchOptions {
+    readonly entity: string;
+    readonly records: ReadonlyArray<StoredValues>;
+}
+export interface UpdateManyInput extends BatchOptions {
+    readonly entity: string;
+    readonly records: ReadonlyArray<{
+        readonly id: string;
+        readonly values: StoredValues;
+    }>;
+}
+export interface UpsertManyInput extends CreateManyInput {
+    readonly on: readonly [string, ...string[]];
+}
+type BatchWriteError = ActionRejected | Conflict | NotFound;
+interface BatchWrite<Input extends BatchOptions> {
+    (input: Input & {
+        readonly returning: "summary";
+    }): Effect.Effect<BatchSummaryResult, BatchWriteError>;
+    (input: Input & {
+        readonly returning?: "records";
+    }): Effect.Effect<BatchRecordResult, BatchWriteError>;
+    (input: Input): Effect.Effect<BatchResult, BatchWriteError>;
+}
 type ReadsService = {
     readonly get: (input: {
         readonly entity: string;
@@ -88,6 +133,9 @@ type ReadsService = {
     }) => Effect.Effect<ReadonlyArray<StoredRecord>>;
 };
 type WritesService = {
+    readonly createMany: BatchWrite<CreateManyInput>;
+    readonly updateMany: BatchWrite<UpdateManyInput>;
+    readonly upsertMany: BatchWrite<UpsertManyInput>;
     readonly create: (input: {
         readonly entity: string;
         readonly values: StoredValues;
@@ -130,7 +178,8 @@ export declare class OntologyReads extends OntologyReads_base {
 declare const OntologyWrites_base: Context.ServiceClass<OntologyWrites, "@segmentstream-cloud/ontology/OntologyWrites", WritesService>;
 /**
  * The writes an action handler runs, inside the action's transaction: `create`, `update`,
- * `upsertBy`, `delete`, `link` and `unlink` of entity handles require it. `prepare` never has it.
+ * `upsertBy`, the batch methods, `delete`, `link` and `unlink` of entity handles require it.
+ * `prepare` never has it.
  */
 export declare class OntologyWrites extends OntologyWrites_base {
 }

package/dist/runtime.js

@@ -8,7 +8,8 @@ export class OntologyReads extends Context.Service()("@segmentstream-cloud/ontol
 }
 /**
  * The writes an action handler runs, inside the action's transaction: `create`, `update`,
- * `upsertBy`, `delete`, `link` and `unlink` of entity handles require it. `prepare` never has it.
+ * `upsertBy`, the batch methods, `delete`, `link` and `unlink` of entity handles require it.
+ * `prepare` never has it.
  */
 export class OntologyWrites extends Context.Service()("@segmentstream-cloud/ontology/OntologyWrites") {
 }

package/dist/service/errors.js

@@ -1,6 +1,7 @@
 import { Cause, Context, Effect, Option, Predicate, Schema, SchemaIssue } from "effect";
 import { HttpServerResponse } from "effect/unstable/http";
 import { HttpApiError } from "effect/unstable/httpapi";
+import { BatchErrorDetails } from "../errors.js";
 /** The correlation id of the current request, in every error body and the `x-request-id` header. */
 export class RequestId extends Context.Service()("@segmentstream-cloud/ontology/RequestId") {
 }
@@ -64,6 +65,19 @@ function listed(values) {
         ? values.join("")
         : `${values.slice(0, -1).join(", ")} and ${values.at(-1)}`;
 }
+/** Only the declared safe locations cross a transport, even for structurally tagged failures. */
+function batchDetails(error) {
+    const batch = error.batch;
+    if (!Schema.is(BatchErrorDetails)(batch)) {
+        return {};
+    }
+    return {
+        batch: {
+            indices: batch.indices,
+            ...(batch.fields === undefined ? {} : { fields: batch.fields }),
+        },
+    };
+}
 export function toServerResponse(response) {
     return HttpServerResponse.jsonUnsafe(errorBodyForWire(response.body), {
         status: response.status,
@@ -323,6 +337,7 @@ export function expectedFailure(error, requestId) {
                 detail: message,
                 requestId,
                 issues: [{ path: [], message }],
+                ...batchDetails(error),
             },
         };
     }
@@ -421,6 +436,7 @@ export function expectedFailure(error, requestId) {
                 requestId,
                 ...(typeof entity === "string" ? { entity } : {}),
                 ...(typeof id === "string" ? { id } : {}),
+                ...batchDetails(error),
             },
         };
     }
@@ -434,6 +450,7 @@ export function expectedFailure(error, requestId) {
                 requestId,
                 ...(typeof entity === "string" ? { entity } : {}),
                 ...(typeof property === "string" ? { property } : {}),
+                ...batchDetails(error),
             },
         };
     }

package/dist/service/schemas.js

@@ -1,6 +1,7 @@
 import { BlobValue } from "../blob.js";
 import { Schema, SchemaTransformation } from "effect";
 import { wireStruct } from "../wire.js";
+import { BatchErrorDetails } from "../errors.js";
 import { AGGREGATE_FILTER_OPERATORS, ENTITY_AGGREGATE_FILTER_OPERATORS, filterValuesIssue, MAXIMUM_FILTER_VALUES, } from "../aggregate-filters.js";
 import { isValueMatched, OntologyManifestWire } from "../manifest.js";
 import { entityQuerySchemas, queryLinks } from "../query.js";
@@ -510,14 +511,17 @@ export const ErrorSchemas = {
     notFound: errorBody("NotFound", 404, ["not_found"], {
         entity: Schema.optionalKey(Schema.String),
         id: Schema.optionalKey(Schema.String),
+        batch: Schema.optionalKey(BatchErrorDetails),
     }),
     conflict: errorBody("Conflict", 409, ["conflict"], {
         entity: Schema.optionalKey(Schema.String),
         property: Schema.optionalKey(Schema.String),
+        batch: Schema.optionalKey(BatchErrorDetails),
     }),
     unsupportedMediaType: errorBody("UnsupportedMediaType", 415, ["unsupported_media_type"]),
     unprocessable: errorBody("Unprocessable", 422, ["invalid_input", "action_rejected"], {
         issues: Schema.Array(ValidationIssue),
+        batch: Schema.optionalKey(BatchErrorDetails),
     }),
     rateLimited: errorBody("RateLimited", 429, ["rate_limited"]),
     internal: errorBody("InternalError", 500, ["internal_error"]),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@segmentstream-cloud/ontology",
-  "version": "0.25.1",
+  "version": "0.26.0",
   "description": "Define a SegmentStream workspace app's ontology: entities, relationships, roles and actions, written with Effect.",
   "license": "Apache-2.0",
   "author": "SegmentStream",
@@ -45,7 +45,9 @@
     "lint": "eslint src tests --config ../../eslint.config.mjs",
     "test": "vitest run",
     "test:postgres": "vitest run --config vitest.postgres.config.ts",
-    "prepack": "npm run build"
+    "prepack": "npm run build",
+    "benchmark:batch": "tsx tests/batch/benchmark.ts",
+    "benchmark:batch:profile": "tsx tests/batch/profile.ts"
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.1148.0",