@sixb/core 0.1.19 → 0.1.21

package/src/lake-storage/version-inputs.ts

@@ -1,12 +1,41 @@
-import type { DatasetVersionRef } from "./types"
+import { COMMITTING_PROVIDER, sameLineage, versionLineage } from "./lineage"
+import type { DatasetVersion } from "./types"
 
-/** Input lineage participates in write identity when the caller supplies it. */
-export function hasDatasetInputChanges(
-  previous: readonly DatasetVersionRef[] | undefined,
-  next: readonly DatasetVersionRef[] | undefined
+/** What a commit would store: `lineage` is the one `lineageAtCommit` returned for it. */
+export type DatasetCommitProvenance = Pick<
+  DatasetVersion,
+  "datasetId" | "schema" | "producer" | "inputs" | "lineage"
+>
+
+/**
+ * Provenance participates in write identity: identical rows computed from other input versions,
+ * or with a lineage that classifies them otherwise, still create a version. Reusing the latest
+ * version would keep provenance that no longer describes the newest calculation.
+ *
+ * `next` holds what the commit would store, after `lineageAtCommit` merged in the parent's
+ * lineage. Lineages compare whatever their order and repetition; an unknown lineage, or a pipeline
+ * version that records none, differs from any known one. Inputs count only when the writer passed
+ * them.
+ */
+export function hasDatasetProvenanceChanges(
+  latest: DatasetCommitProvenance & Pick<DatasetVersion, "versionId">,
+  next: DatasetCommitProvenance
+): boolean {
+  return (
+    hasDatasetInputChanges(latest.inputs, next.inputs) ||
+    !sameLineage(
+      versionLineage(latest, COMMITTING_PROVIDER),
+      versionLineage(next, COMMITTING_PROVIDER)
+    )
+  )
+}
+
+function hasDatasetInputChanges(
+  previous: DatasetVersion["inputs"],
+  next: DatasetVersion["inputs"]
 ): boolean {
   if (next === undefined) return false
-  const keys = (refs: readonly DatasetVersionRef[]) =>
+  const keys = (refs: NonNullable<DatasetVersion["inputs"]>) =>
     new Set(refs.map((ref) => JSON.stringify([ref.datasetId, ref.versionId])))
   const before = keys(previous ?? [])
   const after = keys(next)

package/src/maintenance/ontology-maintenance.ts

@@ -231,6 +231,21 @@ export class OntologyMaintenance {
       )
     }
 
+    // A frame grant lasts an hour; nothing reads one after it expires.
+    const artifacts = this.storage.artifacts
+    if (artifacts) {
+      await captureFailure(
+        () =>
+          artifacts.deleteExpiredFrameGrants({
+            projectId: this.projectId,
+            expiredBefore: started,
+            limit: this.cleanupLimit,
+          }),
+        failures,
+        0
+      )
+    }
+
     return {
       publishedOutboxRowsDeleted,
       terminalSourceRowsDeleted:

package/src/models/execution/embedding.ts

@@ -58,9 +58,9 @@ async function executeEmbedding(
   if (purpose !== undefined && purpose !== "document" && purpose !== "query") {
     throw new TypeError("[SixbModels] Embedding purpose must be 'document' or 'query'.")
   }
+  // The caller owns the deadline: only it knows whether anyone is waiting for the result.
   const signal = session.signal(input.signal)
-  const requestSignal = AbortSignal.any([signal, AbortSignal.timeout(30_000)])
-  requestSignal.throwIfAborted()
+  signal.throwIfAborted()
   if (!texts.length) return { vectors: [] }
 
   const accounting = await session.accounting()
@@ -72,7 +72,7 @@ async function executeEmbedding(
       "[SixbModels] Resolved embedding model identity does not match the selected model."
     )
   }
-  requestSignal.throwIfAborted()
+  signal.throwIfAborted()
   const callId = `call_${randomUUID()}`
   const tokens = texts.reduce(
     (sum, text) => sum + Math.ceil(new TextEncoder().encode(text).byteLength / 4),
@@ -92,8 +92,8 @@ async function executeEmbedding(
 
   let result: EmbeddingModelResult
   try {
-    requestSignal.throwIfAborted()
-    result = await model.embed({ texts, purpose, signal: requestSignal })
+    signal.throwIfAborted()
+    result = await model.embed({ texts, purpose, signal })
   } catch (error) {
     if (error instanceof EmbeddingModelResponseError) {
       await recordEmbedding(accounting, model, callId, error.metadata)
@@ -105,7 +105,7 @@ async function executeEmbedding(
   }
   // Account before cancellation, vector validation or the materializer's optimistic write.
   await recordEmbedding(accounting, model, callId, result ?? {})
-  requestSignal.throwIfAborted()
+  signal.throwIfAborted()
   return result
 }
 

package/src/objects/clearance-redaction.ts

@@ -0,0 +1,138 @@
+import type { StoredDomainEvent } from "../events/types"
+import type { ExpandedLinkValue, ExpandedObjectRow, ObjectRow } from "../storage"
+import type { PropertyClearance } from "./property-clearance"
+
+/**
+ * Omit unreadable properties from a row and the objects expanded under it, and list them in
+ * `redactions`. The row is not modified: what changes is copied, the rest is shared.
+ */
+export function redactObjectRow<TRow extends ObjectRow>(
+  row: TRow,
+  clearance: PropertyClearance
+): TRow {
+  return redactRow(row, clearance, true)
+}
+
+/**
+ * {@link redactObjectRow} without the `redactions` records, for rows about to be deep-copied.
+ *
+ * Omitting hidden values before the copy keeps them out of it and copies each row once. The copy
+ * then receives its records from {@link attachRedactions}: they are frozen and owned by the
+ * clearance, so they are shared rather than copied with each row.
+ */
+export function omitHiddenProperties<TRow extends ObjectRow>(
+  row: TRow,
+  clearance: PropertyClearance
+): TRow {
+  return redactRow(row, clearance, false)
+}
+
+/**
+ * Set `redactions` on a row returned by {@link omitHiddenProperties} and since deep-copied, and on
+ * the objects expanded under it. It modifies the row, so it only accepts the caller's own copy.
+ */
+export function attachRedactions(row: ObjectRow, clearance: PropertyClearance): void {
+  const hidden = clearance.hiddenProperties(row.objectTypeId)
+  if (hidden !== undefined) row.redactions = hidden.redactions
+
+  if (row.links === undefined) return
+
+  for (const value of Object.values(row.links)) {
+    if (value === null) continue
+    if (Array.isArray(value)) {
+      for (const linked of value) attachRedactions(linked, clearance)
+    } else {
+      attachRedactions(value as ExpandedObjectRow, clearance)
+    }
+  }
+}
+
+function redactRow<TRow extends ObjectRow>(
+  row: TRow,
+  clearance: PropertyClearance,
+  withRedactions: boolean
+): TRow {
+  const hidden = clearance.hiddenProperties(row.objectTypeId)
+  const links = row.links
+  if (hidden === undefined && links === undefined) return row
+
+  const redacted: TRow = { ...row }
+  if (hidden !== undefined) {
+    redacted.properties = withoutKeys(row.properties, hidden.ids)
+    if (withRedactions) redacted.redactions = hidden.redactions
+  }
+
+  if (links !== undefined) {
+    redacted.links = Object.fromEntries(
+      Object.entries(links).map(([linkId, value]) => [
+        linkId,
+        redactLinkValue(value, clearance, withRedactions),
+      ])
+    )
+  }
+  return redacted
+}
+
+function redactLinkValue(
+  value: ExpandedLinkValue,
+  clearance: PropertyClearance,
+  withRedactions: boolean
+): ExpandedLinkValue {
+  if (value === null) return null
+  if (Array.isArray(value)) {
+    return value.map((row) => redactRow(row, clearance, withRedactions))
+  }
+
+  // Array.isArray does not narrow a readonly[] away, so assert the single-object case.
+  return redactRow(value as ExpandedObjectRow, clearance, withRedactions)
+}
+
+/**
+ * Omit unreadable properties from an object event, under the same contract as rows. Other events
+ * carry no markable value: link properties and telemetry properties cannot be marked.
+ */
+export function redactDomainEvent(
+  event: StoredDomainEvent,
+  clearance: PropertyClearance
+): StoredDomainEvent {
+  if (event.topic !== "objects") return event
+
+  const hidden = clearance.hiddenProperties(event.payload.objectTypeId)
+  if (hidden === undefined) return event
+
+  if (event.type === "object.deleted") {
+    return {
+      ...event,
+      payload: {
+        ...event.payload,
+        propertyChanges: withoutKeys(event.payload.propertyChanges, hidden.ids),
+        redactions: hidden.redactions,
+      },
+    }
+  }
+
+  return {
+    ...event,
+    payload: {
+      ...event.payload,
+      properties: withoutKeys(event.payload.properties, hidden.ids),
+      propertyChanges: withoutKeys(event.payload.propertyChanges, hidden.ids),
+      redactions: hidden.redactions,
+    },
+  }
+}
+
+/**
+ * A copy of the record's own properties without the hidden keys.
+ *
+ * Spreading the record and deleting the few hidden keys is several times faster than assigning
+ * each kept key, and the spread defines every key as an own property, even `__proto__`.
+ */
+function withoutKeys<TValue>(
+  record: Readonly<Record<string, TValue>>,
+  hidden: ReadonlySet<string>
+): Record<string, TValue> {
+  const kept: Record<string, TValue> = { ...record }
+  for (const key of hidden) delete kept[key]
+  return kept
+}

package/src/objects/link/authorization.ts

@@ -4,8 +4,14 @@
 import { assertAuthorized, assertCanEdit } from "../../authorization"
 import type { SixbRuntimeContext } from "../../runtime/types"
 
+interface LinkWrite {
+  readonly sourceTypeId: string
+  readonly linkId: string
+  readonly targetTypeId: string
+}
+
 /**
- * Assert a principal may create or remove a link.
+ * Assert a principal may create or remove these links.
  *
  * `edit` on the **source** type, `view` on the **target**: a link is declared on the source type
  * (`objectType.links`), and writing one changes the source's edges, not the target's properties.
@@ -18,10 +24,24 @@ import type { SixbRuntimeContext } from "../../runtime/types"
  *
  * Call this *before* any endpoint lookup, for that second reason.
  */
-export function assertCanWriteLink(
-  ctx: Pick<SixbRuntimeContext, "projectId" | "runtimeAuthorization" | "authorization">,
-  endpoints: { readonly sourceTypeId: string; readonly targetTypeId: string }
-): void {
-  assertCanEdit(ctx, endpoints.sourceTypeId)
-  assertAuthorized(ctx, { kind: "object.view", objectTypeId: endpoints.targetTypeId })
+export async function assertCanWriteLinks(
+  ctx: Pick<
+    SixbRuntimeContext,
+    "projectId" | "runtimeAuthorization" | "authorization" | "objectReader"
+  >,
+  writes: readonly LinkWrite[]
+): Promise<void> {
+  for (const write of writes) {
+    assertCanEdit(ctx, write.sourceTypeId)
+    assertAuthorized(ctx, { kind: "object.view", objectTypeId: write.targetTypeId })
+  }
+
+  // An endpoint type or a link the reader cannot read under the markings of the versions
+  // projections applied is refused like one it has no view grant for, for the same reason.
+  await ctx.objectReader.assertReadable({
+    objectTypeIds: [
+      ...new Set(writes.flatMap((write) => [write.sourceTypeId, write.targetTypeId])),
+    ],
+    links: writes.map((write) => ({ objectTypeId: write.sourceTypeId, linkId: write.linkId })),
+  })
 }

package/src/objects/link/remove.ts

@@ -8,7 +8,7 @@ import {
   linkDeleteOperation,
   runtimeOperationId,
 } from "../materializer-adapter"
-import { assertCanWriteLink } from "./authorization"
+import { assertCanWriteLinks } from "./authorization"
 
 export async function removeLink(
   ctx: ResolvedLinkContext,
@@ -21,7 +21,7 @@ export async function removeLink(
 ): Promise<void> {
   const { objectType, linkDefinition, ontology } = ctx
   const { sourceId, linkId, targetTypeId, targetId } = params
-  assertCanWriteLink(ctx, { sourceTypeId: objectType.id, targetTypeId })
+  await assertCanWriteLinks(ctx, [{ sourceTypeId: objectType.id, linkId, targetTypeId }])
 
   assertLinkTargetType(objectType.id, linkId, linkDefinition, targetTypeId, (expected, actual) =>
     ontology.isValidLinkTarget(expected, actual)

package/src/objects/link/upsert-batch.ts

@@ -16,7 +16,7 @@ import {
   runtimeOperationId,
 } from "../materializer-adapter"
 import { runRuntimeItemBatch } from "../runtime-batch"
-import { assertCanWriteLink } from "./authorization"
+import { assertCanWriteLinks } from "./authorization"
 import { collectEndpointLookups, loadEndpointExistence, requireEndpoints } from "./endpoints"
 
 export async function upsertLinkBatch(
@@ -30,12 +30,14 @@ export async function upsertLinkBatch(
   // a per-item validation: a refused item must not become one failed `BatchItemResult` while the
   // rest of the batch commits. Asserting inside `plan` would do exactly that, and would also let the
   // existence lookup answer for targets the principal cannot see.
-  for (const item of items) {
-    assertCanWriteLink(ctx, {
+  await assertCanWriteLinks(
+    ctx,
+    items.map((item) => ({
       sourceTypeId: item.objectType.id,
+      linkId: item.linkId,
       targetTypeId: item.targetTypeId,
-    })
-  }
+    }))
+  )
 
   const existing = await loadEndpointExistence(ctx, collectEndpointLookups(items))
   const valueTypesById = ontology.getValueTypesById()

package/src/objects/link/upsert.ts

@@ -8,7 +8,7 @@ import {
   normalizeRuntimeLink,
   runtimeOperationId,
 } from "../materializer-adapter"
-import { assertCanWriteLink } from "./authorization"
+import { assertCanWriteLinks } from "./authorization"
 import { collectEndpointLookups, loadEndpointExistence, requireEndpoints } from "./endpoints"
 
 export async function upsertLink(
@@ -23,7 +23,7 @@ export async function upsertLink(
 ): Promise<void> {
   const { objectType, linkDefinition, ontology } = ctx
   const { sourceId, linkId, targetTypeId, targetId, properties } = params
-  assertCanWriteLink(ctx, { sourceTypeId: objectType.id, targetTypeId })
+  await assertCanWriteLinks(ctx, [{ sourceTypeId: objectType.id, linkId, targetTypeId }])
   const endpoints = { objectType, sourceId, targetTypeId, targetId }
 
   // Endpoint reads keep the public `ObjectNotFoundError` contract; the Materializer independently

package/src/objects/object/delete.ts

@@ -22,6 +22,7 @@ import {
  */
 export async function deleteObject(ctx: ResolvedObjectContext, primaryId: string): Promise<void> {
   assertCanEdit(ctx, ctx.objectType.id)
+  await ctx.objectReader.assertReadable({ objectTypeIds: [ctx.objectType.id] })
   await commitRuntimeOperations(ctx, [
     objectDeleteOperation({
       id: runtimeOperationId(0),
@@ -38,6 +39,7 @@ export async function deleteObject(ctx: ResolvedObjectContext, primaryId: string
  */
 export async function restoreObject(ctx: ResolvedObjectContext, primaryId: string): Promise<void> {
   assertCanEdit(ctx, ctx.objectType.id)
+  await ctx.objectReader.assertReadable({ objectTypeIds: [ctx.objectType.id] })
   await commitRuntimeOperations(ctx, [
     objectRestoreOperation({
       id: runtimeOperationId(0),

package/src/objects/object/upsert-batch.ts

@@ -25,6 +25,7 @@ export async function upsertObjectBatch(
   assertCanEdit(ctx, ctx.objectType.id)
   const { objectType, primaryPropertyId, ontology } = ctx
   const valueTypesById = ontology.getValueTypesById()
+  const redact = await ctx.objectReader.writtenRowRedactor(objectType.id)
 
   return runRuntimeItemBatch({
     ctx,
@@ -52,9 +53,6 @@ export async function upsertObjectBatch(
         ],
       }
     },
-    value: ([outcome]) =>
-      ctx.objectReader.redactWrittenRow(
-        toObjectRow(ctx.projectId, requireEffectiveObject(outcome))
-      ),
+    value: ([outcome]) => redact(toObjectRow(ctx.projectId, requireEffectiveObject(outcome))),
   })
 }

package/src/objects/object/upsert.ts

@@ -20,6 +20,7 @@ export async function upsertObject(
 ): Promise<ObjectRow> {
   assertCanEdit(ctx, ctx.objectType.id)
   const { objectType, primaryPropertyId, ontology } = ctx
+  const redact = await ctx.objectReader.writtenRowRedactor(objectType.id)
 
   const normalized = normalizeRuntimeObject({
     objectType,
@@ -34,7 +35,5 @@ export async function upsertObject(
   const commit = await commitRuntimeOperations(ctx, [
     objectUpsertOperation({ id: runtimeOperationId(0), ref, properties: normalized.properties }),
   ])
-  return ctx.objectReader.redactWrittenRow(
-    toObjectRow(ctx.projectId, requireEffectiveObject(commit.outcomes[0]))
-  )
+  return redact(toObjectRow(ctx.projectId, requireEffectiveObject(commit.outcomes[0])))
 }