@segmentstream-cloud/ontology 0.20.0 → 0.22.0

package/README.md

@@ -370,7 +370,7 @@ both are recorded, moves the members and a missing `purpose` and `whenToUse` to
 deletes the old one. `provider`, `externalId`, `title` and `lastMessageAt` are
 managed. Members set `purpose` and `whenToUse` with the generated
 `agent_chat.update`, the chat's only generated action. The built-in action
-`agent_chat.send_message` (`{ message, chat?, person? }`, `message` at most 4000 characters) asks the agent to write to the chat
+`agent_chat.send_message` (`{ message, chat?, person?, document_ids? }`, `message` at most 4000 characters) asks the agent to write to the chat
 given, else to the person's one chat, else to their paired private chat; a person in several
 chats is refused with the candidates, or only their number without `agent_chat:read`. The service posts the message to the agent at
 `https://<VERCEL_PROJECT_PRODUCTION_URL>/eve/v1/telegram/messages` with the deployment's OIDC
@@ -382,6 +382,30 @@ chats needs `agent_chat:read` by name: `*:read` and `*:*` do not cover `agent_ch
 
 ### Agent attachments
 
+Ontology SDK 0.21.0 adds `documentIds` to the authored action input (`document_ids` on REST/MCP):
+an optional array of at most ten `document` record UUIDs. Before posting anything to the agent,
+the action checks `document:read`, every record's existence and its single file property.
+Missing files refuse the request; unreadable files require the read permission; ambiguous file
+properties are refused. Requests without the field retain their previous behavior. The service
+requires a paired member requester for requests with documents, since the agent downloads as
+that member; services and unpaired requesters may still send requests without documents. It
+forwards `documentIds` to `POST /eve/v1/telegram/messages`; agent SDK 0.5.0 includes the ids in
+`<agent_request>` and sends the stored bytes with `segmentstream__attachments_send`.
+
+`GET /api/ontology/agent/files/:entity/:id[/:property]` returns `{ file, download }` for the
+current file property of an ontology record. Like temporary attachment routes, it requires a
+production runtime bearer token, `X-SegmentStream-Act-As` naming an active paired member,
+and the agent conversation/session headers. It checks `<entity>:read` before returning metadata
+and rechecks live permissions when granting a download pinned to that revision. The response is
+uncached; the download grant is short lived. Without a property the entity must have exactly one
+file property. Missing records/files return `not_found`, missing read access `forbidden`, and
+non-file or ambiguous properties `unsupported`.
+
+Outbound Telegram uploads currently have an effective **2 MiB multipart request limit** in the
+connection proxy, including overhead, even though Telegram accepts larger files. The agent splits
+compatible albums to fit and returns per-file errors. Group purpose and document restrictions
+still apply; a member's read access alone does not authorize sharing a restricted document in a group.
+
 A file a member sends the agent is a temporary attachment: platform metadata in the runtime
 table `_agent_attachments`, never an ontology record, with its bytes stored once as a revision in
 the ontology's file storage, beside uploads. It belongs to the member, the chat it arrived in and

package/dist/agent-chat.d.ts

@@ -14,6 +14,8 @@ export interface AgentMessage {
     };
     /** The request for the agent, written as an instruction to it. */
     readonly message: string;
+    /** Document record ids to send with the request; absent from older requests. */
+    readonly documentIds?: ReadonlyArray<string>;
     /** Whom the agent should address, when the caller named someone. */
     readonly person: {
         readonly id: string;
@@ -63,6 +65,7 @@ export declare const AgentChat: import("./entity.js").Entity<"agent_chat", {
 }, {
     readonly sendMessage: import("./action.js").Action<Schema.Struct<{
         readonly message: import("./property.js").Property<string, false, false>;
+        readonly documentIds: Schema.optionalKey<Schema.$Array<Schema.String>>;
         readonly chat: Schema.optionalKey<Schema.Codec<string, string, never, never>>;
         readonly person: Schema.optionalKey<Schema.Codec<string, string, never, never>>;
     }>, {

package/dist/agent-chat.js

@@ -6,6 +6,7 @@ import { isAllowed, permissionOf, READ_VERB } from "./permissions.js";
 import { p } from "./property.js";
 import { rel } from "./relationship.js";
 import { User } from "./user.js";
+import { AgentFileRecords } from "./runtime/files.js";
 /** The only channel the agent delivers to so far, and the provider of its chats. */
 export const TELEGRAM = "telegram";
 /**
@@ -84,12 +85,16 @@ export const AgentChat = entity("agent_chat", {
             "chats is refused with the candidates (id, title, purpose, when to use; only their number " +
             "without agent_chat:read): call again with chat. The agent then writes in that chat and " +
             "handles the replies there, in that chat's own session; this action only delivers the " +
-            "request and does not wait for an answer. Requires an agent deployed in the workspace.",
+            "request and does not wait for an answer. document_ids optionally names up to ten readable " +
+            "document records to attach with the question. Requires an agent deployed in the workspace.",
         input: Schema.Struct({
             message: p.text({
                 maxLength: 4000,
                 description: "The request for the agent, as an instruction: what to ask or tell, of whom.",
             }),
+            documentIds: Schema.optionalKey(Schema.Array(Schema.String.check(Schema.isUUID()))
+                .check(Schema.isMaxLength(10))
+                .annotate({ description: "Up to ten document record ids the requester may read." })),
             chat: p.ref("agent_chat", { optional: true, description: "The agent chat to write in." }),
             person: p.ref("user", {
                 optional: true,
@@ -103,6 +108,19 @@ export const AgentChat = entity("agent_chat", {
             if (Option.isNone(channel)) {
                 return yield* new ActionRejected({ message: NO_AGENT });
             }
+            if (input.documentIds !== undefined && input.documentIds.length > 0) {
+                const files = yield* Effect.serviceOption(AgentFileRecords);
+                if (Option.isNone(files)) {
+                    return yield* new ActionRejected({
+                        message: "This runtime cannot validate stored documents.",
+                    });
+                }
+                for (const id of input.documentIds) {
+                    yield* files.value
+                        .resolve({ entity: "document", id }, principal)
+                        .pipe(Effect.catchTag("UnsupportedFile", (error) => Effect.fail(new ActionRejected({ message: error.message }))));
+                }
+            }
             const person = input.person === undefined ? undefined : yield* User.get(input.person);
             if (person !== undefined && (person.kind !== "member" || !person.active)) {
                 return yield* new ActionRejected({
@@ -158,9 +176,15 @@ export const AgentChat = entity("agent_chat", {
             // The agent acts as a member only through their pairing.
             const actAs = principal.kind === "member" &&
                 (yield* channel.value.privateChat(principal.id)) !== undefined;
+            if ((input.documentIds?.length ?? 0) > 0 && !actAs) {
+                return yield* new ActionRejected({
+                    message: "Sending documents requires a member requester with a paired Telegram account.",
+                });
+            }
             yield* channel.value.send({
                 target,
                 message: input.message,
+                ...(input.documentIds === undefined ? {} : { documentIds: input.documentIds }),
                 person: person === undefined ? null : { id: person.id, name: person.name },
                 requester: {
                     id: requester.id,

package/dist/migrate/apply.js

@@ -1,13 +1,12 @@
-import { randomUUID } from "node:crypto";
 import { Duration, Effect, Schema } from "effect";
 import { canonicalJson, decodeManifestDocument, encodeManifest, manifestHash, } from "../manifest.js";
 import { CatalogInvalid, CatalogMismatch, MigrationFailed, MigrationRefused } from "./errors.js";
 import { isNoopMigration } from "./model.js";
 import { planMigration } from "./plan.js";
 import { REPLICATION_DDL, REPLICATION_SCHEMA_VERSION, replicationEntityDdl, } from "../replication/schema.js";
-import { createReplicationBaseline } from "../replication/baseline.js";
+import { replicationMigrationRefusals } from "../replication/migrate.js";
 import { ATTACHMENTS_TABLE } from "../storage/tables.js";
-import { resolveReplicationSelection } from "../replication/model.js";
+import { resolveReplicationSelection, } from "../replication/model.js";
 import { AGENT_TABLES, API_KEYS_TABLE, CATALOG_TABLE, OAUTH_TABLES, USER_ROLES_TABLE, quoteIdentifier, quoteLiteral, } from "./storage.js";
 const CATALOG = quoteIdentifier(CATALOG_TABLE);
 /** Serializes migrations of one database; transaction-scoped, so it ends with the transaction. */
@@ -122,8 +121,25 @@ export const applyMigration = (plan, options) => Effect.gen(function* () {
         if (presence?.present) {
             const [state] = yield* sql.unsafe("SELECT * FROM _ontology_replication WHERE id FOR UPDATE");
             if (state?.enabled) {
-                refresh = state.exclusions ?? [];
-                // Writers wait on this row. Capture, schema changes and the new baseline commit together.
+                refresh = yield* Effect.try({
+                    try: () => resolveReplicationSelection(plan.target, state.exclusions ?? []),
+                    catch: (cause) => new CatalogInvalid({ message: "The replication exclusions are invalid.", cause }),
+                });
+                const [target] = yield* sql.unsafe("SELECT * FROM _ontology_replication_targets WHERE generation = $1::uuid", [state.active_generation]);
+                if (!target) {
+                    return yield* new CatalogInvalid({
+                        message: "The active replication target is missing; pause and rebuild replication.",
+                        cause: { generation: state.active_generation },
+                    });
+                }
+                const refusals = replicationMigrationRefusals(target, plan.target, refresh);
+                if (refusals.length > 0) {
+                    return yield* new MigrationRefused({
+                        message: refusals.map((refusal) => `${refusal.path}: ${refusal.message}`).join(" "),
+                        refusals,
+                    });
+                }
+                // Writers wait on this row. Capture and schema changes commit together on the same stream.
                 yield* sql.unsafe("UPDATE _ontology_replication SET enabled = false WHERE id");
             }
         }
@@ -148,14 +164,9 @@ export const applyMigration = (plan, options) => Effect.gen(function* () {
               "applied_at" = now()
           `, [plan.target.version, plan.target.hash, canonicalJson(encodeManifest(plan.target))]);
         if (refresh !== undefined) {
-            const selection = yield* Effect.sync(() => resolveReplicationSelection(plan.target, refresh));
-            yield* createReplicationBaseline(sql, {
-                generation: randomUUID(),
-                manifest: plan.target,
-                exclusions: refresh,
-                selection,
-                newSource: true,
-            });
+            yield* sql.unsafe(`UPDATE _ontology_replication_targets SET manifest = $1::jsonb, selection = $2::jsonb
+             WHERE generation = (SELECT active_generation FROM _ontology_replication WHERE id)`, [JSON.stringify(plan.target), JSON.stringify(refresh)]);
+            yield* sql.unsafe("UPDATE _ontology_replication SET selection = $1::jsonb, enabled = true WHERE id", [JSON.stringify(Object.fromEntries(refresh.map((item) => [item.entityId, item])))]);
         }
     }));
     return { hash: plan.target.hash, appliedSteps: plan.steps.length, changed: true };

package/dist/migrate/errors.d.ts

@@ -2,7 +2,7 @@ import { Schema } from "effect";
 declare const MigrationRefused_base: Schema.Class<MigrationRefused, Schema.TaggedStruct<"MigrationRefused", {
     readonly message: Schema.String;
     readonly refusals: Schema.$Array<Schema.Struct<{
-        readonly code: Schema.Literals<readonly ["invalid-manifest", "type-change", "relationship-change", "drop-refused", "name-collision", "identifier-too-long", "rename-conflict"]>;
+        readonly code: Schema.Literals<readonly ["invalid-manifest", "type-change", "relationship-change", "drop-refused", "name-collision", "identifier-too-long", "rename-conflict", "replication-rebuild-required"]>;
         readonly path: Schema.String;
         readonly message: Schema.String;
     }>>;

package/dist/migrate/model.d.ts

@@ -25,11 +25,11 @@ export declare const MigrationStep: Schema.Struct<{
     readonly sql: Schema.$Array<Schema.String>;
 }>;
 export type MigrationStep = typeof MigrationStep.Type;
-export declare const MIGRATION_REFUSAL_CODES: readonly ["invalid-manifest", "type-change", "relationship-change", "drop-refused", "name-collision", "identifier-too-long", "rename-conflict"];
+export declare const MIGRATION_REFUSAL_CODES: readonly ["invalid-manifest", "type-change", "relationship-change", "drop-refused", "name-collision", "identifier-too-long", "rename-conflict", "replication-rebuild-required"];
 export type MigrationRefusalCode = (typeof MIGRATION_REFUSAL_CODES)[number];
 /** A change the planner will not make. A plan with refusals cannot be applied. */
 export declare const MigrationRefusal: Schema.Struct<{
-    readonly code: Schema.Literals<readonly ["invalid-manifest", "type-change", "relationship-change", "drop-refused", "name-collision", "identifier-too-long", "rename-conflict"]>;
+    readonly code: Schema.Literals<readonly ["invalid-manifest", "type-change", "relationship-change", "drop-refused", "name-collision", "identifier-too-long", "rename-conflict", "replication-rebuild-required"]>;
     /** Where, in the style of `check` diagnostics: `lead.properties.status`. */
     readonly path: Schema.String;
     readonly message: Schema.String;
@@ -284,7 +284,7 @@ export declare const MigrationPlan: Schema.Struct<{
         readonly sql: Schema.$Array<Schema.String>;
     }>>;
     readonly refusals: Schema.$Array<Schema.Struct<{
-        readonly code: Schema.Literals<readonly ["invalid-manifest", "type-change", "relationship-change", "drop-refused", "name-collision", "identifier-too-long", "rename-conflict"]>;
+        readonly code: Schema.Literals<readonly ["invalid-manifest", "type-change", "relationship-change", "drop-refused", "name-collision", "identifier-too-long", "rename-conflict", "replication-rebuild-required"]>;
         /** Where, in the style of `check` diagnostics: `lead.properties.status`. */
         readonly path: Schema.String;
         readonly message: Schema.String;

package/dist/migrate/model.js

@@ -55,6 +55,7 @@ export const MIGRATION_REFUSAL_CODES = [
     "name-collision",
     "identifier-too-long",
     "rename-conflict",
+    "replication-rebuild-required",
 ];
 /** A change the planner will not make. A plan with refusals cannot be applied. */
 export const MigrationRefusal = Schema.Struct({

package/dist/replication/baseline.d.ts

@@ -1,4 +1,4 @@
-import { Effect } from "effect";
+import { Effect, Schema } from "effect";
 import type { SqlClient } from "effect/unstable/sql";
 import type { OntologyManifest } from "../manifest.js";
 import { type ReplicationSelection, type ReplicationExclusion } from "./model.js";
@@ -9,6 +9,13 @@ export interface ReplicationTarget {
     readonly manifest: OntologyManifest;
     readonly selection: ReadonlyArray<ReplicationSelection>;
 }
+declare const ReplicationBaselineFailed_base: Schema.Class<ReplicationBaselineFailed, Schema.TaggedStruct<"ReplicationBaselineFailed", {
+    readonly message: Schema.String;
+    readonly cause: Schema.optionalKey<Schema.Unknown>;
+}>, import("effect/Cause").YieldableError>;
+/** An initial snapshot cannot be created with the requested state or record projection. */
+export declare class ReplicationBaselineFailed extends ReplicationBaselineFailed_base {
+}
 /** All baseline rows are committed with capture enablement; interrupted setup can safely resume. */
 export declare const createReplicationBaseline: (sql: SqlClient.SqlClient, input: {
     readonly generation: string;
@@ -16,4 +23,5 @@ export declare const createReplicationBaseline: (sql: SqlClient.SqlClient, input
     readonly exclusions?: ReadonlyArray<ReplicationExclusion>;
     readonly selection: ReadonlyArray<ReplicationSelection>;
     readonly newSource?: boolean;
-}) => Effect.Effect<ReplicationTarget, import("effect/unstable/sql/SqlError").SqlError, never>;
+}) => Effect.Effect<ReplicationTarget, ReplicationBaselineFailed | import("effect/unstable/sql/SqlError").SqlError, never>;
+export {};

package/dist/replication/baseline.js

@@ -1,26 +1,35 @@
 import { randomUUID } from "node:crypto";
-import { Effect } from "effect";
+import { Effect, Schema } from "effect";
 import { joinColumnNames, joinTableName, quoteIdentifier as qi, quoteLiteral as ql, tableName, } from "../migrate/storage.js";
 import { REPLICATION_RECORD_BYTES, validateReplicationSelection, } from "./model.js";
 import { snapshotExpression } from "./schema.js";
-const MAX_BASELINE_RECORDS = 50_000;
+/** An initial snapshot cannot be created with the requested state or record projection. */
+export class ReplicationBaselineFailed extends Schema.TaggedError()("ReplicationBaselineFailed", { message: Schema.String, cause: Schema.optionalKey(Schema.Unknown) }) {
+}
 /** All baseline rows are committed with capture enablement; interrupted setup can safely resume. */
 export const createReplicationBaseline = (sql, input) => sql.withTransaction(Effect.gen(function* () {
     yield* sql.unsafe("SET LOCAL statement_timeout = '30s'");
     yield* sql.unsafe("SET LOCAL lock_timeout = '5s'");
     const [state] = yield* sql.unsafe("SELECT source_id::text, revision::text, enabled FROM _ontology_replication WHERE id FOR UPDATE");
     if (!state) {
-        return yield* Effect.die(new Error("Run the ontology SDK migration before enabling replication."));
+        return yield* new ReplicationBaselineFailed({
+            message: "Run the ontology SDK migration before enabling replication.",
+        });
     }
     const existing = yield* sql.unsafe("SELECT * FROM _ontology_replication_targets WHERE generation = $1::uuid", [input.generation]);
     if (existing[0]) {
         return existing[0];
     }
     if (state.enabled) {
-        return yield* Effect.die(new Error("Pause replication before starting a new generation."));
+        return yield* new ReplicationBaselineFailed({
+            message: "Pause replication before starting a new generation.",
+        });
     }
     const manifest = input.manifest;
-    yield* Effect.sync(() => validateReplicationSelection(manifest, input.selection));
+    yield* Effect.try({
+        try: () => validateReplicationSelection(manifest, input.selection),
+        catch: (cause) => new ReplicationBaselineFailed({ message: "Invalid replication projection.", cause }),
+    });
     // Drain older SDK transactions that started before the compatibility fence was enabled.
     const tables = manifest.entities
         .filter((e) => e.source === undefined && !e.builtIn)
@@ -57,17 +66,17 @@ export const createReplicationBaseline = (sql, input) => sql.withTransaction(Eff
         JSON.stringify(manifest),
         JSON.stringify(input.selection),
     ]);
-    let total = 0;
     const baseline = (entityId, kind, records) => Effect.gen(function* () {
         // Keep the snapshot in PostgreSQL; large baselines must not fill a serverless process's heap.
-        const [size] = yield* sql.unsafe(`SELECT count(*)::int AS count, max(octet_length(payload::text)) AS largest
-          FROM (${records} LIMIT ${MAX_BASELINE_RECORDS + 1}) snapshot`);
-        total += size.count;
-        if (total > MAX_BASELINE_RECORDS) {
-            return yield* Effect.die(new Error("The replication baseline exceeds 50,000 records; use a bulk bootstrap."));
-        }
-        if ((size.largest ?? 0) > REPLICATION_RECORD_BYTES) {
-            return yield* Effect.die(new Error("A projected baseline record exceeds the replication size limit."));
+        const [size] = yield* sql.unsafe(`SELECT EXISTS (
+              SELECT 1 FROM (${records}) snapshot
+              WHERE octet_length(payload::text) > ${REPLICATION_RECORD_BYTES}
+            ) AS oversized`);
+        if (size.oversized) {
+            return yield* new ReplicationBaselineFailed({
+                message: `A projected baseline record exceeds ${REPLICATION_RECORD_BYTES} bytes; ` +
+                    "exclude large fields before enabling replication.",
+            });
         }
         yield* sql.unsafe(`INSERT INTO _ontology_replication_baselines (generation, event)
       SELECT $1::uuid, jsonb_build_object('source_id', $2::text, 'entity_id', $3::text,

package/dist/replication/migrate.d.ts

@@ -0,0 +1,6 @@
+import type { OntologyManifest } from "../manifest.js";
+import type { MigrationRefusal } from "../migrate/model.js";
+import type { ReplicationTarget } from "./baseline.js";
+import type { ReplicationSelection } from "./model.js";
+/** A live stream can change its projection only when retained JSON still describes current rows. */
+export declare function replicationMigrationRefusals(current: ReplicationTarget, manifest: OntologyManifest, selection: ReadonlyArray<ReplicationSelection>): ReadonlyArray<MigrationRefusal>;

package/dist/replication/migrate.js

@@ -0,0 +1,64 @@
+import { toSnakeCase } from "../names.js";
+import { queryFields } from "../query.js";
+/** A live stream can change its projection only when retained JSON still describes current rows. */
+export function replicationMigrationRefusals(current, manifest, selection) {
+    const refusals = [];
+    const refuse = (path, reason) => {
+        refusals.push({
+            code: "replication-rebuild-required",
+            path,
+            message: `${reason} Pause replication, apply the migration, then explicitly rebuild replication.`,
+        });
+    };
+    for (const previous of current.selection) {
+        const entity = current.manifest.entities.find((item) => item.id === previous.entityId);
+        const next = selection.find((item) => item.entityId === entity.id);
+        if (!next) {
+            refuse(entity.name, "Removing a replicated entity would discard its pending history.");
+            continue;
+        }
+        for (const field of previous.fields) {
+            if (!next.fields.includes(field)) {
+                refuse(`${entity.name}.${field}`, "Removing or renaming a replicated field would leave retained JSON under its old name.");
+            }
+        }
+        for (const relationship of previous.relationships ?? []) {
+            if (!next.relationships?.includes(relationship)) {
+                refuse(`${entity.name}.${relationship}`, "Removing a replicated relationship would leave its old edges in retained JSON.");
+            }
+        }
+    }
+    for (const next of selection) {
+        const entity = manifest.entities.find((item) => item.id === next.entityId);
+        const previousEntity = current.manifest.entities.find((item) => item.id === entity.id);
+        // A newly created table starts empty and all subsequent writes are captured normally.
+        if (!previousEntity) {
+            continue;
+        }
+        const previous = current.selection.find((item) => item.entityId === entity.id);
+        if (!previous) {
+            refuse(entity.name, "Previously excluded records are absent from retained JSON.");
+            continue;
+        }
+        const previousFields = new Set(queryFields(previousEntity).map((field) => field.name));
+        for (const field of next.fields.filter((name) => !previous.fields.includes(name))) {
+            const property = entity.properties.find((item) => item.name === field);
+            const renamed = property?.renamedFrom;
+            if (previousFields.has(field) ||
+                (renamed !== undefined &&
+                    (previousFields.has(renamed) || previousFields.has(toSnakeCase(renamed))))) {
+                refuse(`${entity.name}.${field}`, "Existing values are absent from this JSON field.");
+            }
+            else if (property?.default !== undefined && property.default !== null) {
+                refuse(`${entity.name}.${field}`, "Adding a field with a non-null default changes existing rows without captured writes.");
+            }
+        }
+        for (const relationship of next.relationships ?? []) {
+            if (!previous.relationships?.includes(relationship) &&
+                previousEntity.relationships.some((item) => item.name === relationship)) {
+                refuse(`${entity.name}.${relationship}`, "Previously excluded relationship edges are absent from retained JSON.");
+            }
+        }
+    }
+    return refusals;
+}

package/dist/replication/model.d.ts

@@ -1,5 +1,5 @@
 import { type OntologyManifest } from "../manifest.js";
-/** The resolved projection for one immutable generation. `id` is always included. */
+/** The current projection of a continuous replication stream. `id` is always included. */
 export interface ReplicationSelection {
     readonly entityId: string;
     readonly fields: ReadonlyArray<string>;

package/dist/replication/store.d.ts

@@ -1,14 +1,14 @@
 import { Effect } from "effect";
 import type { SqlClient } from "effect/unstable/sql";
 import { type ReplicationEvent, type ReplicationSelection, type ReplicationExclusion } from "./model.js";
-import { type ReplicationTarget } from "./baseline.js";
+import { ReplicationBaselineFailed, type ReplicationTarget } from "./baseline.js";
 export type { ReplicationTarget } from "./baseline.js";
 export declare const startReplication: (sql: SqlClient.SqlClient, input: {
     readonly generation: string;
     readonly exclusions?: ReadonlyArray<ReplicationExclusion>;
     readonly selection?: ReadonlyArray<ReplicationSelection>;
     readonly newSource?: boolean;
-}) => Effect.Effect<ReplicationTarget, import("../index.js").CatalogInvalid | import("effect/unstable/sql/SqlError").SqlError, never>;
+}) => Effect.Effect<ReplicationTarget, import("../index.js").CatalogInvalid | ReplicationBaselineFailed | import("effect/unstable/sql/SqlError").SqlError, never>;
 export declare const activeReplicationTarget: (sql: SqlClient.SqlClient) => Effect.Effect<ReplicationTarget, import("effect/unstable/sql/SqlError").SqlError, never>;
 export declare const pauseReplication: (sql: SqlClient.SqlClient) => import("effect/unstable/sql/Statement").Statement<object>;
 export declare const replicationTarget: (sql: SqlClient.SqlClient, generation: string) => Effect.Effect<ReplicationTarget, import("effect/unstable/sql/SqlError").SqlError, never>;

package/dist/replication/store.js

@@ -2,17 +2,22 @@ import { Effect } from "effect";
 import { readCatalog } from "../migrate/apply.js";
 import { historyTableName, quoteIdentifier as qi } from "../migrate/storage.js";
 import { REPLICATION_BATCH_BYTES, REPLICATION_BATCH_SIZE, publishMessage, replicationEventId, resolveReplicationSelection, } from "./model.js";
-import { createReplicationBaseline } from "./baseline.js";
+import { createReplicationBaseline, ReplicationBaselineFailed, } from "./baseline.js";
 export const startReplication = (sql, input) => sql.withTransaction(Effect.gen(function* () {
     yield* sql.unsafe("SET LOCAL statement_timeout = '30s'");
     yield* sql.unsafe("SET LOCAL lock_timeout = '5s'");
     yield* sql.unsafe("SELECT id FROM _ontology_replication WHERE id FOR UPDATE");
     const manifest = yield* readCatalog({ sql });
     if (!manifest) {
-        return yield* Effect.die(new Error("The ontology has not been migrated."));
+        return yield* new ReplicationBaselineFailed({
+            message: "The ontology has not been migrated.",
+        });
     }
     const selection = input.selection ??
-        (yield* Effect.sync(() => resolveReplicationSelection(manifest, input.exclusions)));
+        (yield* Effect.try({
+            try: () => resolveReplicationSelection(manifest, input.exclusions),
+            catch: (cause) => new ReplicationBaselineFailed({ message: "Invalid replication exclusions.", cause }),
+        }));
     return yield* createReplicationBaseline(sql, { ...input, manifest, selection });
 }));
 export const activeReplicationTarget = (sql) => sql

package/dist/runtime/files.d.ts

@@ -0,0 +1,39 @@
+import { Context, Effect, Schema } from "effect";
+import { NotFound, PermissionDenied } from "../errors.js";
+import type { OntologyManifest } from "../manifest.js";
+import type { OntologyReads, Principal } from "../runtime.js";
+export interface FileReference {
+    readonly entity: string;
+    readonly id: string;
+    readonly property?: string;
+}
+declare const UnsupportedFile_base: Schema.Class<UnsupportedFile, Schema.TaggedStruct<"UnsupportedFile", {
+    readonly message: Schema.String;
+}>, import("effect/Cause").YieldableError>;
+export declare class UnsupportedFile extends UnsupportedFile_base {
+}
+/** Resolve only declared file properties, checking permission before reading any record. */
+export declare function resolveFileRecord(input: {
+    readonly manifest: OntologyManifest;
+    readonly reads: Pick<OntologyReads["Service"], "get">;
+    readonly principal: Principal;
+    readonly reference: FileReference;
+}): Effect.Effect<{
+    entity: string;
+    id: string;
+    property: string;
+    file: {
+        readonly id: string;
+        readonly filename: string;
+        readonly contentType: string;
+        readonly size: number;
+        readonly createdAt: string;
+    };
+}, PermissionDenied | NotFound | UnsupportedFile, never>;
+declare const AgentFileRecords_base: Context.ServiceClass<AgentFileRecords, "@segmentstream-cloud/ontology/AgentFileRecords", {
+    readonly resolve: (reference: FileReference, principal: Principal) => ReturnType<typeof resolveFileRecord>;
+}>;
+/** File validation in an action's prepare, bound to its ontology and read runtime. */
+export declare class AgentFileRecords extends AgentFileRecords_base {
+}
+export {};

package/dist/runtime/files.js

@@ -0,0 +1,44 @@
+import { Context, Effect, Schema } from "effect";
+import { BlobValue } from "../blob.js";
+import { NotFound, PermissionDenied } from "../errors.js";
+import { isAllowed, permissionOf, READ_VERB } from "../permissions.js";
+export class UnsupportedFile extends Schema.TaggedError()("UnsupportedFile", {
+    message: Schema.String,
+}) {
+}
+/** Resolve only declared file properties, checking permission before reading any record. */
+export function resolveFileRecord(input) {
+    return Effect.gen(function* () {
+        const { entity: entityName, id, property: propertyName } = input.reference;
+        const permission = permissionOf(entityName, READ_VERB);
+        if (!isAllowed(permission, input.principal)) {
+            return yield* new PermissionDenied({ permission, message: "The file requires read access." });
+        }
+        const entity = input.manifest.entities.find((candidate) => candidate.name === entityName);
+        if (entity === undefined) {
+            return yield* new NotFound({ entity: entityName, id, message: "File record not found." });
+        }
+        const properties = entity.properties.filter((candidate) => candidate.type === "blob");
+        const property = propertyName === undefined
+            ? properties.length === 1
+                ? properties[0]
+                : undefined
+            : properties.find((candidate) => candidate.name === propertyName);
+        if (property === undefined) {
+            return yield* new UnsupportedFile({
+                message: "Name a file property; the entity must have exactly one when property is omitted.",
+            });
+        }
+        const record = yield* input.reads.get({ entity: entityName, id });
+        const file = record[property.name];
+        if (file === null || file === undefined) {
+            return yield* new NotFound({ entity: entityName, id, message: "File not found." });
+        }
+        // Runtime records are decoded already; malformed stored metadata is an operator failure.
+        const decoded = yield* Schema.decodeUnknownEffect(BlobValue)(file).pipe(Effect.orDie);
+        return { entity: entityName, id, property: property.name, file: decoded };
+    });
+}
+/** File validation in an action's prepare, bound to its ontology and read runtime. */
+export class AgentFileRecords extends Context.Service()("@segmentstream-cloud/ontology/AgentFileRecords") {
+}

package/dist/runtime/pipeline.js

@@ -14,6 +14,7 @@ import { UnknownAction, ValidationIssue } from "./errors.js";
 import { generatedActions } from "./generated-actions.js";
 import { Edit } from "./history.js";
 import { makeOntologyRuntime } from "./sql-runtime.js";
+import { AgentFileRecords, resolveFileRecord } from "./files.js";
 import { validate } from "./validation.js";
 /** How long `prepare` may take, connection calls included; each call has its own 30 s deadline. */
 export const PREPARE_TIMEOUT = Duration.seconds(60);
@@ -243,7 +244,9 @@ function prepare(ontology, sql, action, context, connections, qualified) {
             find: ({ entity, options }) => runtime.find({ entity, options: { ...options, forUpdate: false } }),
         };
         const run = Effect.suspend(() => action.prepare(context));
-        return yield* run.pipe(Effect.provideService(OntologyReads, reads), Effect.provideContext(connectionContext(action.connections, transport ?? NO_CONNECTIONS)), Effect.timeoutOrElse({
+        return yield* run.pipe(Effect.provideService(OntologyReads, reads), Effect.provideService(AgentFileRecords, {
+            resolve: (reference, principal) => resolveFileRecord({ manifest: ontology.manifest, reads, reference, principal }),
+        }), Effect.provideContext(connectionContext(action.connections, transport ?? NO_CONNECTIONS)), Effect.timeoutOrElse({
             duration: PREPARE_TIMEOUT,
             orElse: () => Effect.fail(new ConnectionError({
                 connection: action.connections.map((handle) => handle.key).join(","),

package/dist/service/agent/attachments.d.ts

@@ -7,8 +7,9 @@ import { RequestId } from "../errors.js";
 import type { ApiPlan } from "../plan.js";
 import type { ReplicationKick } from "../replication.js";
 import { OntologyBackend } from "../runtime-contract.js";
-import { type RuntimeTokenVerifier } from "../runtime-identity.js";
+import type { RuntimeTokenVerifier } from "../runtime-identity.js";
 import type { AgentChannelFactory } from "./channel.js";
+export { CONVERSATION_HEADER, SESSION_HEADER } from "./authentication.js";
 /**
  * The agent's temporary attachments over HTTP (`docs/agent.md`, "Attachments"). Only the
  * deployment's runtime token reaches these routes, acting as the member in `X-SegmentStream-Act-As`
@@ -17,10 +18,6 @@ import type { AgentChannelFactory } from "./channel.js";
  * tool arguments; every operation is confined to that member, chat and session.
  */
 export declare const ATTACHMENTS_PATH = "/api/ontology/agent/attachments";
-/** The chat an attachment belongs to, such as `telegram:12345`. */
-export declare const CONVERSATION_HEADER = "x-segmentstream-agent-conversation";
-/** The agent session the request comes from. */
-export declare const SESSION_HEADER = "x-segmentstream-agent-session";
 export interface AttachmentRoutesOptions {
     readonly plan: ApiPlan;
     readonly hooks: ReadonlyMap<string, AfterCommitHook>;

package/dist/service/agent/attachments.js

@@ -1,14 +1,14 @@
-import { Cause, Effect, Layer, Redacted, Result, Schema } from "effect";
-import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
-import { HttpApiBuilder, HttpApiSecurity } from "effect/unstable/httpapi";
+import { Cause, Effect, Layer, Result, Schema } from "effect";
+import { HttpRouter, HttpServerResponse } from "effect/unstable/http";
 import { ATTACHMENT_SOURCES, } from "../../storage/attachments.js";
 import { schemaIssues } from "../../runtime/validation.js";
 import { CurrentPrincipal } from "../authentication.js";
-import { RequestId, errorBodyForWire, failureResponse, forbidden, toServerResponse, unauthorized, unavailable, } from "../errors.js";
+import { RequestId, errorBodyForWire, failureResponse } from "../errors.js";
 import { identityToken, makeOperations } from "../operations.js";
 import { OntologyBackend } from "../runtime-contract.js";
-import { ACT_AS_HEADER, logRuntimeRejection, } from "../runtime-identity.js";
+import { authenticateAgentCaller } from "./authentication.js";
 import { AGENT_PATH } from "./server.js";
+export { CONVERSATION_HEADER, SESSION_HEADER } from "./authentication.js";
 /**
  * The agent's temporary attachments over HTTP (`docs/agent.md`, "Attachments"). Only the
  * deployment's runtime token reaches these routes, acting as the member in `X-SegmentStream-Act-As`
@@ -17,12 +17,6 @@ import { AGENT_PATH } from "./server.js";
  * tool arguments; every operation is confined to that member, chat and session.
  */
 export const ATTACHMENTS_PATH = `${AGENT_PATH}/attachments`;
-/** The chat an attachment belongs to, such as `telegram:12345`. */
-export const CONVERSATION_HEADER = "x-segmentstream-agent-conversation";
-/** The agent session the request comes from. */
-export const SESSION_HEADER = "x-segmentstream-agent-session";
-const CONVERSATION = /^[a-z][a-z0-9_-]{0,31}:[A-Za-z0-9:_-]{1,200}$/;
-const SESSION = /^[A-Za-z0-9:_.-]{1,200}$/;
 const NO_STORE = { "cache-control": "no-store" };
 const Received = Schema.Struct({
     source: Schema.Literals(ATTACHMENT_SOURCES),
@@ -111,39 +105,16 @@ export function attachmentsLayer(options) {
             return;
         }
         const operations = makeOperations(backend, options.plan, options.hooks, options.connections, options.replicate, options.agentChannel);
-        const credential = HttpApiBuilder.securityDecode(HttpApiSecurity.bearer);
         /** Authenticates the runtime, then the member it acts for, the chat and the session. */
         const asAgent = (answer) => (request) => Effect.gen(function* () {
-            const requestId = yield* RequestId;
-            const bearer = yield* Effect.provideService(credential, HttpServerRequest.HttpServerRequest, request);
-            const token = Redacted.value(bearer);
-            const verified = token === "" ? undefined : yield* Effect.promise(() => verifyRuntime(token));
-            switch (verified?.status) {
-                case "valid":
-                    break;
-                case "invalid":
-                    yield* logRuntimeRejection(verified.reason);
-                    return toServerResponse(unauthorized(requestId, "The bearer token is invalid, expired or revoked."));
-                case "unavailable":
-                    return toServerResponse(unavailable(requestId));
-                default:
-                    return toServerResponse(forbidden(requestId, "Only the app's agent calls this route."));
-            }
-            const actAs = request.headers[ACT_AS_HEADER]?.trim() ?? "";
-            const conversation = request.headers[CONVERSATION_HEADER]?.trim() ?? "";
-            const session = request.headers[SESSION_HEADER]?.trim() ?? "";
-            const member = actAs === "" ? undefined : yield* agents.actingMember(actAs);
-            if (member === undefined) {
-                return toServerResponse(forbidden(requestId, "The member is inactive or has not paired with the agent."));
-            }
-            if (!CONVERSATION.test(conversation) || (session !== "" && !SESSION.test(session))) {
-                return toServerResponse(forbidden(requestId, "The agent names the chat and session of every request."));
-            }
-            return yield* answer(request, {
-                principal: member,
-                conversation,
-                ...(session === "" ? {} : { sessionId: session }),
-            }, attachments);
+            const authenticated = yield* authenticateAgentCaller({
+                request,
+                agents,
+                verify: verifyRuntime,
+            });
+            return authenticated.ok
+                ? yield* answer(request, authenticated.caller, attachments)
+                : authenticated.response;
         });
         const idOf = Effect.map(HttpRouter.params, (params) => params.id ?? "");
         yield* router.add("POST", ATTACHMENTS_PATH, asAgent((request, caller, runtime) => answered(Effect.gen(function* () {

package/dist/service/agent/authentication.d.ts

@@ -0,0 +1,20 @@
+import { Effect } from "effect";
+import { HttpServerRequest } from "effect/unstable/http";
+import type { AttachmentCaller } from "../../storage/attachments.js";
+import { RequestId } from "../errors.js";
+import type { AgentStore } from "../runtime-contract.js";
+import { type RuntimeTokenVerifier } from "../runtime-identity.js";
+export declare const CONVERSATION_HEADER = "x-segmentstream-agent-conversation";
+export declare const SESSION_HEADER = "x-segmentstream-agent-session";
+/** Both temporary attachments and stored files require the runtime and its paired member. */
+export declare function authenticateAgentCaller(input: {
+    readonly request: HttpServerRequest.HttpServerRequest;
+    readonly agents: AgentStore;
+    readonly verify: RuntimeTokenVerifier;
+}): Effect.Effect<{
+    readonly ok: false;
+    readonly response: import("effect/unstable/http/HttpServerResponse").HttpServerResponse;
+} | {
+    readonly ok: true;
+    readonly caller: AttachmentCaller;
+}, never, RequestId | HttpServerRequest.ParsedSearchParams>;

package/dist/service/agent/authentication.js

@@ -0,0 +1,48 @@
+import { Effect, Redacted } from "effect";
+import { HttpServerRequest } from "effect/unstable/http";
+import { HttpApiBuilder, HttpApiSecurity } from "effect/unstable/httpapi";
+import { RequestId, forbidden, toServerResponse, unauthorized, unavailable } from "../errors.js";
+import { ACT_AS_HEADER, logRuntimeRejection, } from "../runtime-identity.js";
+export const CONVERSATION_HEADER = "x-segmentstream-agent-conversation";
+export const SESSION_HEADER = "x-segmentstream-agent-session";
+const CONVERSATION = /^[a-z][a-z0-9_-]{0,31}:[A-Za-z0-9:_-]{1,200}$/;
+const SESSION = /^[A-Za-z0-9:_.-]{1,200}$/;
+/** Both temporary attachments and stored files require the runtime and its paired member. */
+export function authenticateAgentCaller(input) {
+    return Effect.gen(function* () {
+        const { request, agents, verify } = input;
+        const requestId = yield* RequestId;
+        const credential = HttpApiBuilder.securityDecode(HttpApiSecurity.bearer);
+        const bearer = yield* Effect.provideService(credential, HttpServerRequest.HttpServerRequest, request);
+        const token = Redacted.value(bearer);
+        const verified = token === "" ? undefined : yield* Effect.promise(() => verify(token));
+        const denied = (response) => ({ ok: false, response: toServerResponse(response) });
+        switch (verified?.status) {
+            case "valid":
+                break;
+            case "invalid":
+                yield* logRuntimeRejection(verified.reason);
+                return denied(unauthorized(requestId, "The bearer token is invalid, expired or revoked."));
+            case "unavailable":
+                return denied(unavailable(requestId));
+            default:
+                return denied(forbidden(requestId, "Only the app's agent calls this route."));
+        }
+        const actAs = request.headers[ACT_AS_HEADER]?.trim() ?? "";
+        const conversation = request.headers[CONVERSATION_HEADER]?.trim() ?? "";
+        const session = request.headers[SESSION_HEADER]?.trim() ?? "";
+        const member = actAs === "" ? undefined : yield* agents.actingMember(actAs);
+        if (member === undefined) {
+            return denied(forbidden(requestId, "The member is inactive or has not paired with the agent."));
+        }
+        if (!CONVERSATION.test(conversation) || (session !== "" && !SESSION.test(session))) {
+            return denied(forbidden(requestId, "The agent names the chat and session of every request."));
+        }
+        const caller = {
+            principal: member,
+            conversation,
+            ...(session === "" ? {} : { sessionId: session }),
+        };
+        return { ok: true, caller };
+    });
+}

package/dist/service/agent/files.d.ts

@@ -0,0 +1,12 @@
+import { Layer } from "effect";
+import { HttpRouter } from "effect/unstable/http";
+import { RequestId } from "../errors.js";
+import type { ApiPlan } from "../plan.js";
+import { OntologyBackend } from "../runtime-contract.js";
+import type { RuntimeTokenVerifier } from "../runtime-identity.js";
+export declare const FILES_PATH = "/api/ontology/agent/files";
+/** A short-lived download grant, issued only after the runtime's member may read the file. */
+export declare function filesLayer(options: {
+    readonly plan: ApiPlan;
+    readonly verifyRuntime: RuntimeTokenVerifier | undefined;
+}): Layer.Layer<never, never, OntologyBackend | HttpRouter.HttpRouter | HttpRouter.Request<"Requires", RequestId>>;

package/dist/service/agent/files.js

@@ -0,0 +1,69 @@
+import { Cause, Effect, Layer } from "effect";
+import { HttpRouter, HttpServerResponse } from "effect/unstable/http";
+import { resolveFileRecord } from "../../runtime/files.js";
+import { isUuid } from "../../query.js";
+import { RequestId, failureResponse, notFound, toServerResponse } from "../errors.js";
+import { OntologyBackend } from "../runtime-contract.js";
+import { authenticateAgentCaller } from "./authentication.js";
+import { AGENT_PATH } from "./server.js";
+export const FILES_PATH = `${AGENT_PATH}/files`;
+/** A short-lived download grant, issued only after the runtime's member may read the file. */
+export function filesLayer(options) {
+    return Layer.effectDiscard(Effect.gen(function* () {
+        const router = yield* HttpRouter.HttpRouter;
+        const backend = yield* OntologyBackend;
+        const { blobs, agents } = backend;
+        const verify = options.verifyRuntime;
+        if (blobs === undefined || agents === undefined || verify === undefined) {
+            return;
+        }
+        const read = (request) => Effect.gen(function* () {
+            const requestId = yield* RequestId;
+            const authenticated = yield* authenticateAgentCaller({ request, agents, verify });
+            if (!authenticated.ok) {
+                return authenticated.response;
+            }
+            const params = yield* HttpRouter.params;
+            const id = params.id ?? "";
+            if (!isUuid(id)) {
+                return toServerResponse(notFound(requestId, "File record not found."));
+            }
+            const resolved = yield* resolveFileRecord({
+                manifest: options.plan.ontology.manifest,
+                reads: backend,
+                principal: authenticated.caller.principal,
+                reference: {
+                    entity: params.entity ?? "",
+                    id,
+                    ...(params.property === undefined ? {} : { property: params.property }),
+                },
+            });
+            // Pin the revision read above. download rechecks live permissions in its transaction.
+            const download = yield* blobs.download(resolved.entity, resolved.id, resolved.property, { revisionId: resolved.file.id }, authenticated.caller.principal);
+            return HttpServerResponse.jsonUnsafe({
+                file: {
+                    id,
+                    entity: resolved.entity,
+                    property: resolved.property,
+                    filename: resolved.file.filename,
+                    content_type: resolved.file.contentType,
+                    size: resolved.file.size,
+                },
+                download: {
+                    url: download.url,
+                    method: download.method,
+                    headers: download.headers,
+                    expires_at: download.expiresAt,
+                },
+            }, { headers: { "cache-control": "no-store" } });
+        }).pipe(Effect.catchTag("UnsupportedFile", (error) => Effect.map(RequestId, (requestId) => HttpServerResponse.jsonUnsafe({
+            error: "unsupported",
+            detail: error.message,
+            request_id: requestId,
+        }, { status: 422 }))), Effect.catchCause((cause) => Cause.hasInterruptsOnly(cause)
+            ? Effect.failCause(cause)
+            : Effect.flatMap(RequestId, (requestId) => failureResponse(cause, requestId))));
+        yield* router.add("GET", `${FILES_PATH}/:entity/:id`, read);
+        yield* router.add("GET", `${FILES_PATH}/:entity/:id/:property`, read);
+    }));
+}

package/dist/service/agent/server.js

@@ -170,7 +170,13 @@ function seeChat(store) {
         if (Result.isFailure(decoded)) {
             return invalidBody(requestId, schemaIssues(decoded.failure));
         }
-        return json(yield* store.seeChat(decoded.success));
+        const chat = yield* store.seeChat(decoded.success);
+        yield* Effect.logInfo("ontology.agent.chat_reported").pipe(Effect.annotateLogs({
+            requestId,
+            provider: decoded.success.provider,
+            chatId: decoded.success.externalId,
+        }));
+        return json(chat);
     });
 }
 /** `GET /api/ontology/agent/identities` (member): the member's own live identities. */

package/dist/service/service.js

@@ -5,6 +5,7 @@ import { storageMaintenanceAuthenticationLayer } from "../storage/maintenance-au
 import { nameToWire } from "../authoring-names.js";
 import { sqlServiceRuntime } from "../runtime/service-binding.js";
 import { attachmentsLayer } from "./agent/attachments.js";
+import { filesLayer } from "./agent/files.js";
 import { platformConversations } from "./agent/conversations.js";
 import { conversationRoutes } from "./agent/conversation-routes.js";
 import { deployedAgentChannel } from "./agent/channel.js";
@@ -130,7 +131,7 @@ export function createOntologyService(ontology, config) {
         verifyToken,
         verifyRuntime,
     });
-    const application = Layer.mergeAll(routes, mcp, agent, attachments, conversations === undefined
+    const application = Layer.mergeAll(routes, mcp, agent, attachments, filesLayer({ plan: built.plan, verifyRuntime }), conversations === undefined
         ? Layer.empty
         : conversationRoutes({
             transport: conversations,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@segmentstream-cloud/ontology",
-  "version": "0.20.0",
+  "version": "0.22.0",
   "description": "Define a SegmentStream workspace app's ontology: entities, relationships, roles and actions, written with Effect.",
   "license": "Apache-2.0",
   "author": "SegmentStream",