@segmentstream-cloud/ontology 0.17.0 → 0.18.0

package/README.md

@@ -40,6 +40,10 @@ take the `notEquals`, `notIn`, `gt`, `gte`, `lt`, `lte`, `contains`, `startsWith
 SDK 0.14.0 lets the app's own agent call the service as a paired member (see
 [Agents](#agents)). Its migration adds the agent's runtime tables.
 
+SDK 0.18.0 lets blob properties store any file. `accept` is optional and can restrict a
+property to exact MIME types or ranges. Existing explicit restrictions keep their meaning.
+See [File storage](#file-storage).
+
 SDK 0.12.0 adds warehouse replication enabled by default for all stored business entities,
 fields and business relationships. Existing history captures immutable snapshots and the
 service wakes the publisher after commits. Upgrade and redeploy to activate it; workspace
@@ -73,6 +77,62 @@ SDK 0.13.0 also serves aggregates. Upgrade notes:
 See [ontology aggregates](https://github.com/segmentstream/segmentstream/blob/main/docs/ontology-aggregates.md)
 and [ontology analytics](https://github.com/segmentstream/segmentstream/blob/main/docs/ontology-aggregates-spec.md).
 
+## File storage
+
+Use `p.blob({ description: "Original source file." })` to retain a file's original bytes.
+Without `accept`, a blob property accepts every concrete MIME type, including custom formats
+and `application/octet-stream`. `p.blob()` also defaults to all files, though ontology checks
+require a description. The manifest publishes this policy as `accept: ["*/*"]`.
+
+Restrict a property when its business rules require a format:
+
+```ts
+content: p.blob({
+  description: "Original POS report or bank statement.",
+  accept: [
+    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    "application/vnd.ms-excel",
+    "text/csv",
+    "text/tab-separated-values",
+  ],
+  maxBytes: 10 * 1024 * 1024,
+}),
+```
+
+`accept` takes MIME types, not extensions. Matching ignores case; `image/*` accepts image
+types and `*/*` accepts all types. Empty lists, duplicate entries, invalid MIME types and
+parameters such as `; charset=utf-8` are rejected during ontology checks.
+`BLOB_CONTENT_TYPES` lists common types as a convenience; it is not a global allow-list.
+
+| Source format         | MIME type                                                                 |
+| --------------------- | ------------------------------------------------------------------------- |
+| XLSX                  | `application/vnd.openxmlformats-officedocument.spreadsheetml.sheet`       |
+| XLS                   | `application/vnd.ms-excel`                                                |
+| CSV                   | `text/csv`                                                                |
+| TSV                   | `text/tab-separated-values`                                               |
+| JSON                  | `application/json`                                                        |
+| TXT                   | `text/plain`                                                              |
+| DOCX                  | `application/vnd.openxmlformats-officedocument.wordprocessingml.document` |
+| ZIP                   | `application/zip`                                                         |
+| Unknown binary format | `application/octet-stream`                                                |
+
+Uploads still require a valid concrete MIME type, a nonempty file within the property's
+`maxBytes` and the 25 MiB service limit, available workspace quota and action permissions.
+Completion verifies the stored MIME metadata, exact byte count and upload-session binding.
+PDF, JPEG and PNG retain their prefix-signature checks; plain text retains its UTF-8 prefix
+check. Other formats are stored as opaque bytes, without parsing, conversion, archive
+extraction or malware scanning. A MIME declaration does not prove what a file contains.
+Downloads use attachment disposition; only JPEG and PNG may request inline display.
+Agent attachment promotion honors the same `accept` and `maxBytes` restrictions.
+
+Upgrade the ontology SDK to 0.18.0 or later, update any existing `accept` list that should
+allow more types, then check, migrate and redeploy the ontology. The app SDK storage client
+0.13.1 or later uses `application/octet-stream` when the browser supplies no MIME type.
+REST callers should supply that type themselves for unknown formats.
+
+See [blob storage](https://github.com/segmentstream/segmentstream/blob/main/docs/ontology-blob-storage-implementation.md)
+for upload tickets, transfer grants and retained revisions.
+
 ## Define the ontology
 
 ```ts

package/dist/analyze.js

@@ -1,7 +1,7 @@
 import { checkWarehouseSource } from "./warehouse-backed.js";
 import { Schema } from "effect";
 import { BLOB_UPLOAD_ANNOTATION } from "./blob.js";
-import { BLOB_CONTENT_TYPES, MAX_BLOB_BYTES } from "./blob.js";
+import { isBlobAccept, MAX_BLOB_BYTES } from "./blob.js";
 import { isAction } from "./action.js";
 import { CONNECTION_KEY, isConnection } from "./connection.js";
 import { isEntity } from "./entity.js";
@@ -411,12 +411,13 @@ function checkProperty(path, schema, definition, report) {
             report("invalid-property", path, "Blob properties cannot be unique or have a default.");
         }
         if (!definition.accept?.length ||
-            definition.accept.some((type) => !BLOB_CONTENT_TYPES.includes(type)) ||
-            new Set(definition.accept).size !== definition.accept.length ||
+            definition.accept.some((type) => !isBlobAccept(type)) ||
+            new Set(definition.accept.map((type) => type.toLowerCase())).size !==
+                definition.accept.length ||
             !Number.isInteger(definition.maxBytes) ||
             definition.maxBytes < 1 ||
             definition.maxBytes > MAX_BLOB_BYTES) {
-            report("invalid-constraint", path, "Use supported content types and a limit from 1 byte to 25 MiB.");
+            report("invalid-constraint", path, "Use MIME types or ranges and a limit from 1 byte to 25 MiB.");
         }
     }
     if (definition.type === "select") {

package/dist/blob.d.ts

@@ -20,7 +20,14 @@ export declare const BlobUpload: Schema.Struct<{
 export type BlobUpload = typeof BlobUpload.Type;
 export declare const BLOB_UPLOAD_ANNOTATION = "x-ontology-upload";
 export declare const MAX_BLOB_BYTES: number;
-export declare const BLOB_CONTENT_TYPES: readonly ["application/pdf", "image/jpeg", "image/png", "text/plain"];
+/** Common content types for file properties; any valid MIME type may be declared. */
+export declare const BLOB_CONTENT_TYPES: readonly ["application/pdf", "image/jpeg", "image/png", "text/plain", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/vnd.ms-excel", "text/csv", "text/tab-separated-values", "application/json", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "application/zip", "application/octet-stream"];
+/** A concrete MIME type, without wildcards, parameters or header control characters. */
+export declare function isBlobContentType(contentType: string): boolean;
+/** A property's MIME restriction: a concrete type, a type range, or all files. */
+export declare function isBlobAccept(accept: string): boolean;
+/** Omitted restrictions accept every concrete MIME type; an empty list accepts nothing. */
+export declare function acceptsBlobContentType(accept: ReadonlyArray<string> | undefined, contentType: string): boolean;
 type SchemaValue<S> = S extends {
     readonly Type: infer T;
 } ? T : S extends Schema.optionalKey<infer Inner> ? Inner["Type"] : never;

package/dist/blob.js

@@ -28,12 +28,46 @@ function uploadCodec(entity, property) {
 export const BlobUpload = Schema.Struct({ uploadId: Schema.String.check(Schema.isUUID()) });
 export const BLOB_UPLOAD_ANNOTATION = "x-ontology-upload";
 export const MAX_BLOB_BYTES = 25 * 1024 * 1024;
+/** Common content types for file properties; any valid MIME type may be declared. */
 export const BLOB_CONTENT_TYPES = [
     "application/pdf",
     "image/jpeg",
     "image/png",
     "text/plain",
+    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    "application/vnd.ms-excel",
+    "text/csv",
+    "text/tab-separated-values",
+    "application/json",
+    "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    "application/zip",
+    "application/octet-stream",
 ];
+const CONTENT_TYPE = /^[a-z0-9][a-z0-9!#$&^_.+-]*\/[a-z0-9][a-z0-9!#$&^_.+-]*$/i;
+const CONTENT_TYPE_RANGE = /^[a-z0-9][a-z0-9!#$&^_.+-]*\/\*$/i;
+/** A concrete MIME type, without wildcards, parameters or header control characters. */
+export function isBlobContentType(contentType) {
+    return contentType === contentType.trim() && CONTENT_TYPE.test(contentType);
+}
+/** A property's MIME restriction: a concrete type, a type range, or all files. */
+export function isBlobAccept(accept) {
+    return (accept === accept.trim() &&
+        (accept === "*/*" || CONTENT_TYPE_RANGE.test(accept) || isBlobContentType(accept)));
+}
+/** Omitted restrictions accept every concrete MIME type; an empty list accepts nothing. */
+export function acceptsBlobContentType(accept, contentType) {
+    if (!isBlobContentType(contentType)) {
+        return false;
+    }
+    const normalized = contentType.toLowerCase();
+    return (accept === undefined ||
+        accept.some((candidate) => {
+            const pattern = candidate.toLowerCase();
+            return (pattern === "*/*" ||
+                pattern === normalized ||
+                (CONTENT_TYPE_RANGE.test(pattern) && normalized.startsWith(pattern.slice(0, -1))));
+        }));
+}
 /** Internal wire codec for session tickets and generated action inputs. */
 export function uploadInput(entity, property) {
     return uploadCodec(entity, property);

package/dist/property.d.ts

@@ -1,5 +1,5 @@
 import { Schema } from "effect";
-import { BlobValue, type BLOB_CONTENT_TYPES } from "./blob.js";
+import { BlobValue } from "./blob.js";
 /** The closed set of storage types. Each has a known column, JSON Schema, filters and rendering. */
 export declare const PROPERTY_TYPES: readonly ["text", "email", "url", "integer", "number", "boolean", "date", "timestamp", "select", "json", "blob"];
 export type PropertyType = (typeof PROPERTY_TYPES)[number];
@@ -131,10 +131,11 @@ declare function ref<const O extends ReferenceOptions>(target: string | {
 }, options?: O): ReferenceOf<O>;
 /** Property constructors. */
 export interface BlobOptions extends Omit<PropertyOptions<BlobValue>, "default" | "unique"> {
-    readonly accept: ReadonlyArray<(typeof BLOB_CONTENT_TYPES)[number]>;
+    /** MIME types or ranges such as image/*; omitted means any file. */
+    readonly accept?: ReadonlyArray<string>;
     readonly maxBytes?: number;
 }
-declare function blob<const O extends BlobOptions>(options: O): PropertyOf<BlobValue, O>;
+declare function blob<const O extends BlobOptions>(options?: O): PropertyOf<BlobValue, O>;
 export declare const p: {
     readonly text: typeof text;
     readonly email: typeof email;

package/dist/property.js

@@ -181,10 +181,15 @@ function ref(target, options) {
     return (settings.optional === true ? Schema.optionalKey(schema) : schema);
 }
 function blob(options) {
+    const settings = options ?? {};
     return construct({
         type: "blob",
         base: BlobValue,
-        options: { ...options, maxBytes: options.maxBytes ?? MAX_BLOB_BYTES },
+        options: {
+            ...settings,
+            accept: settings.accept ?? ["*/*"],
+            maxBytes: settings.maxBytes ?? MAX_BLOB_BYTES,
+        },
     });
 }
 export const p = {

package/dist/storage/attachments.js

@@ -1,6 +1,6 @@
 import { createHash, randomUUID } from "node:crypto";
 import { Effect } from "effect";
-import { BLOB_UPLOAD_ANNOTATION, MAX_BLOB_BYTES } from "../blob.js";
+import { acceptsBlobContentType, BLOB_UPLOAD_ANNOTATION, MAX_BLOB_BYTES } from "../blob.js";
 import { ActionRejected, Conflict, NotFound } from "../errors.js";
 import { canonicalJson } from "../manifest.js";
 import { actionInputJsonSchema } from "../runtime/generated-actions.js";
@@ -357,7 +357,7 @@ export function attachmentRuntime(ontology, sql, provider) {
             if (row.state !== "ready" || row.expired) {
                 return yield* unavailable("The attachment expired or was discarded.");
             }
-            if (!property.accept?.includes(row.content_type) ||
+            if (!acceptsBlobContentType(property.accept, row.content_type) ||
                 Number(row.size) > (property.maxBytes ?? MAX_BLOB_BYTES)) {
                 return yield* new ActionRejected({
                     message: `${field.entity}.${field.property} accepts ${(property.accept ?? []).join(", ")} ` +

package/dist/storage/sessions.js

@@ -1,6 +1,6 @@
 import { randomUUID } from "node:crypto";
 import { Effect, Schema } from "effect";
-import { uploadInput, BLOB_UPLOAD_ANNOTATION, MAX_BLOB_BYTES } from "../blob.js";
+import { acceptsBlobContentType, uploadInput, BLOB_UPLOAD_ANNOTATION, MAX_BLOB_BYTES, } from "../blob.js";
 import { wireStruct } from "../wire.js";
 import { ActionRejected, Conflict, NotFound } from "../errors.js";
 import { authorizeBlobOperation } from "./authorization.js";
@@ -50,7 +50,7 @@ function allowsUpload(schema, entity, property) {
 }
 function sniff(bytes, contentType) {
     const starts = (magic) => magic.every((byte, index) => bytes[index] === byte);
-    switch (contentType) {
+    switch (contentType.toLowerCase()) {
         case "application/pdf":
             return starts([37, 80, 68, 70, 45]);
         case "image/png":
@@ -67,7 +67,8 @@ function sniff(bytes, contentType) {
                 return false;
             }
         default:
-            return false;
+            // Other MIME types describe opaque bytes; storage does not parse their formats.
+            return true;
     }
 }
 /** Persisted phases surround every provider call; no external I/O holds a database transaction. */
@@ -108,7 +109,7 @@ export function blobRuntime(ontology, sql, provider) {
             entity.source ||
             !property ||
             !found ||
-            !property.accept?.includes(input.contentType) ||
+            !acceptsBlobContentType(property.accept, input.contentType) ||
             input.size > (property.maxBytes ?? MAX_BLOB_BYTES) ||
             Boolean(input.recordId) === Boolean(input.createIntent) ||
             /[\x00-\x1f\x7f/\\]/.test(input.filename)) {
@@ -334,7 +335,8 @@ export function blobRuntime(ontology, sql, provider) {
             key: blob.key,
             filename: blob.filename,
             contentType: blob.content_type,
-            inline: input.inline === true && blob.content_type.startsWith("image/"),
+            inline: input.inline === true &&
+                ["image/jpeg", "image/png"].includes(blob.content_type.toLowerCase()),
         });
     });
     return { begin, status, part, complete, cancel, download };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@segmentstream-cloud/ontology",
-  "version": "0.17.0",
+  "version": "0.18.0",
   "description": "Define a SegmentStream workspace app's ontology: entities, relationships, roles and actions, written with Effect.",
   "license": "Apache-2.0",
   "author": "SegmentStream",