@sixb/core 0.0.1 → 0.1.1

package/src/http/api-routes.ts

@@ -159,6 +159,13 @@ export const SIXB_API_ROUTES: readonly SixbApiRoute[] = [
     accessToken: true,
     agentApi: true,
   },
+  {
+    operationId: "listObjectLinks",
+    method: "GET",
+    path: "/api/objects/:objectTypeId/:objectId/links",
+    accessToken: true,
+    agentApi: true,
+  },
   // The four write routes. Bearer-capable because they now enforce scoped authorization
   // (`edit:object` / `append:telemetry`), which is exactly the precondition this table's rule states.
   // NOT agent-proxied: letting an agent turn write objects on its own is a separate decision, and
@@ -252,6 +259,20 @@ export const SIXB_API_ROUTES: readonly SixbApiRoute[] = [
     accessToken: true,
     agentApi: true,
   },
+  {
+    operationId: "uploadFileRaw",
+    method: "POST",
+    path: "/api/files",
+    accessToken: true,
+    agentApi: true,
+  },
+  {
+    operationId: "listActionRuns",
+    method: "GET",
+    path: "/api/action-runs",
+    accessToken: true,
+    agentApi: true,
+  },
   {
     operationId: "getActionRun",
     method: "GET",
@@ -278,21 +299,35 @@ export const SIXB_API_ROUTES: readonly SixbApiRoute[] = [
     method: "GET",
     path: "/api/workflows",
     accessToken: true,
-    agentApi: false,
+    agentApi: true,
   },
   {
     operationId: "getWorkflow",
     method: "GET",
     path: "/api/workflows/:workflowId",
     accessToken: true,
-    agentApi: false,
+    agentApi: true,
   },
   {
     operationId: "requestWorkflowRun",
     method: "POST",
     path: "/api/workflows/:workflowId/runs",
     accessToken: true,
-    agentApi: false,
+    agentApi: true,
+  },
+  {
+    operationId: "listWorkflowRuns",
+    method: "GET",
+    path: "/api/workflow-runs",
+    accessToken: true,
+    agentApi: true,
+  },
+  {
+    operationId: "getWorkflowRun",
+    method: "GET",
+    path: "/api/workflow-runs/:runId",
+    accessToken: true,
+    agentApi: true,
   },
   {
     operationId: "cancelWorkflowRun",
@@ -313,7 +348,7 @@ export const SIXB_API_ROUTES: readonly SixbApiRoute[] = [
     method: "GET",
     path: "/api/workflow-runs/:runId/files/content",
     accessToken: true,
-    agentApi: false,
+    agentApi: true,
   },
   {
     operationId: "headWorkflowRunFileContent",

package/src/index.ts

@@ -140,9 +140,12 @@ export type {
   DatasetColumnTypeOf,
   DatasetColumnUnionOf,
   DatasetDefinition,
+  DatasetPrimaryKey,
   DatasetSchema,
+  MergeChange,
 } from "./datasets"
 export {
+  change,
   col,
   DatasetValidationError,
   defineDataset,
@@ -154,7 +157,7 @@ export {
 
 export type { Broker, BrokerCursor } from "./broker"
 export { InMemoryBroker } from "./broker"
-export type { JsonValue } from "./json"
+export type { JsonValue, ReadonlyJsonValue } from "./json"
 export {
   assertJsonValue,
   cloneJsonValue,
@@ -629,6 +632,7 @@ export type {
   RequestSyncRunInput,
   SyncBuilder,
   SyncDefinition,
+  SyncMode,
   SyncReadBuilder,
   SyncReadContext,
   SyncReadHandler,
@@ -867,10 +871,21 @@ export type {
   AgentTextPart,
   AgentToolCallPart,
   AgentToolCallState,
+  AgentToolDefinition,
+  AgentToolDescriptionBuilder,
+  AgentToolHandler,
+  AgentToolHandlerResult,
+  AgentToolInputBuilder,
+  AgentToolInputSchema,
+  AgentToolRunBuilder,
+  AgentToolRunContext,
+  AgentToolRunInfo,
   AgentUiMessage,
   AgentUiMessagePart,
   AgentUiToolPart,
   DefineAgentConfig,
+  InferAgentToolInput,
+  InferAgentToolInputSchema,
   RequestAgentRunInput,
   RequestAgentRunResult,
 } from "./agents"
@@ -878,9 +893,11 @@ export {
   AGENT_REASONING_LEVELS,
   AgentDefinitionError,
   AgentRequestError,
+  AgentToolPublicError,
   agentContext,
   agentContextIdentity,
   defineAgent,
+  defineAgentTool,
   isAgentDefinition,
   MAX_AGENT_APP_STATE_ENTRY_BYTES,
   MAX_AGENT_APP_STATE_TOTAL_BYTES,

package/src/json.ts

@@ -3,6 +3,13 @@ export type JsonObject = { [key: string]: JsonValue }
 export type JsonArray = JsonValue[]
 export type JsonValue = JsonPrimitive | JsonObject | JsonArray
 
+/** JSON-compatible input that also accepts readonly containers from typed application code. */
+export type ReadonlyJsonObject = {
+  readonly [key: string]: ReadonlyJsonValue
+}
+export type ReadonlyJsonArray = readonly ReadonlyJsonValue[]
+export type ReadonlyJsonValue = JsonPrimitive | ReadonlyJsonObject | ReadonlyJsonArray
+
 export function isJsonValue(value: unknown): value is JsonValue {
   return getInvalidJsonValueReason(value) === undefined
 }
@@ -14,8 +21,8 @@ export function assertJsonValue(value: unknown, label = "value"): asserts value
   }
 }
 
-export function cloneJsonValue(value: JsonValue): JsonValue {
-  assertJsonValue(value)
+export function cloneJsonValue(value: ReadonlyJsonValue, label = "value"): JsonValue {
+  assertJsonValue(value, label)
   return JSON.parse(JSON.stringify(value)) as JsonValue
 }
 
@@ -74,7 +81,7 @@ export function getInvalidJsonValueReason(
     return undefined
   }
 
-  if (isPlainObject(value)) {
+  if (isPlainRecord(value)) {
     if (seen.has(value)) {
       return `${label} contains a circular reference`
     }
@@ -101,7 +108,7 @@ export function getInvalidJsonValueReason(
   return `${label} is ${describeNonJsonValue(value)}`
 }
 
-function isPlainObject(value: unknown): value is Record<string, unknown> {
+export function isPlainRecord(value: unknown): value is Record<string, unknown> {
   if (typeof value !== "object" || value === null || Array.isArray(value)) {
     return false
   }

package/src/lake-storage/definition-updates.ts

@@ -36,6 +36,7 @@ export function planDatasetDefinitionUpdate(
   requested: DatasetDefinition
 ): DatasetDefinitionUpdatePlan {
   assertSameDataset(existing, requested)
+  assertPrimaryKeyUnchanged(existing, requested)
 
   const addedColumns = getAddedColumnsAfterValidatingExistingColumns(existing, requested)
   assertAddedColumnsAreNullable(requested.id, addedColumns)
@@ -98,6 +99,24 @@ function assertSameDataset(existing: DatasetDefinition, requested: DatasetDefini
   }
 }
 
+function assertPrimaryKeyUnchanged(
+  existing: DatasetDefinition,
+  requested: DatasetDefinition
+): void {
+  const existingPrimaryKey = existing.primaryKey
+  const requestedPrimaryKey = requested.primaryKey
+  const unchanged =
+    existingPrimaryKey === undefined || requestedPrimaryKey === undefined
+      ? existingPrimaryKey === requestedPrimaryKey
+      : areDefinitionValuesEqual(existingPrimaryKey, requestedPrimaryKey)
+
+  if (!unchanged) {
+    throw new LakeStorageError(
+      `[SixbLake] Dataset '${requested.id}' cannot be redefined with an incompatible primaryKey. Primary keys are immutable.`
+    )
+  }
+}
+
 function getAddedColumnsAfterValidatingExistingColumns(
   existing: DatasetDefinition,
   requested: DatasetDefinition
@@ -193,6 +212,7 @@ function mergeDatasetDefinition(
   )
 
   const columns = [...cloneColumns(existing.schema.columns), ...cloneColumns(addedColumns)]
+  const primaryKey = existing.primaryKey
   const partitionBy = requested.partitionBy ?? existing.partitionBy
   const description = requested.description ?? existing.description
 
@@ -200,6 +220,7 @@ function mergeDatasetDefinition(
     kind: "dataset",
     id: requested.id,
     schema: { columns },
+    ...(primaryKey !== undefined ? { primaryKey: structuredClone(primaryKey) } : {}),
     ...(partitionBy !== undefined ? { partitionBy: [...partitionBy] } : {}),
     ...(description !== undefined ? { description } : {}),
   }

package/src/lake-storage/in-memory.ts

@@ -1,8 +1,20 @@
 import { randomUUID } from "node:crypto"
-import type { DatasetDefinition, DatasetSchema } from "../datasets"
+import type { DatasetDefinition, DatasetSchema, MergeChange } from "../datasets"
 import { getDatasetRowValidationError } from "../datasets/validation"
 import { mergeStrictDatasetDefinition } from "./definition-updates"
 import { LakeStorageError } from "./errors"
+import type {
+  BeginDatasetMergeInput,
+  CommitDatasetMergeInput,
+  DatasetMergeCommitResult,
+  LakeMergeSession,
+} from "./merge"
+import {
+  cloneDatasetMergeChange,
+  encodeDatasetPrimaryKey,
+  getDatasetMergeChangeValidationError,
+  getDatasetPrimaryKeyColumns,
+} from "./merge-validation"
 import type {
   BeginDatasetWriteInput,
   CommitDatasetWriteInput,
@@ -74,6 +86,7 @@ function errorMessage(error: unknown): string {
 
 class InMemoryLakeWriteSession implements LakeWriteSession {
   private readonly rows: DatasetRow[] = []
+  private readonly primaryKeys = new Set<string>()
   private closed = false
 
   constructor(
@@ -84,13 +97,30 @@ class InMemoryLakeWriteSession implements LakeWriteSession {
   async writeRows(rows: Iterable<DatasetRow> | AsyncIterable<DatasetRow>): Promise<void> {
     this.assertOpen()
 
+    const stagedRows: DatasetRow[] = []
+    const stagedPrimaryKeys = new Set<string>()
     for await (const row of rows) {
       const validationError = getDatasetRowValidationError(row, this.input.dataset)
       if (validationError) {
         throw new LakeStorageError(`[LakeStorage] ${validationError}`)
       }
 
-      this.rows.push(cloneRow(row))
+      if (getDatasetPrimaryKeyColumns(this.input.dataset) !== null) {
+        const primaryKey = encodeDatasetPrimaryKey(this.input.dataset, row)
+        if (this.primaryKeys.has(primaryKey) || stagedPrimaryKeys.has(primaryKey)) {
+          throw new LakeStorageError(
+            `[LakeStorage] Dataset '${this.input.dataset.id}' write contains duplicate primary key ${primaryKey}.`
+          )
+        }
+        stagedPrimaryKeys.add(primaryKey)
+      }
+
+      stagedRows.push(cloneRow(row))
+    }
+
+    this.rows.push(...stagedRows)
+    for (const primaryKey of stagedPrimaryKeys) {
+      this.primaryKeys.add(primaryKey)
     }
   }
 
@@ -116,11 +146,63 @@ class InMemoryLakeWriteSession implements LakeWriteSession {
   }
 }
 
+class InMemoryLakeMergeSession implements LakeMergeSession {
+  private readonly changes: MergeChange<DatasetRow, DatasetRow>[] = []
+  private closed = false
+
+  constructor(
+    private readonly storage: InMemoryLakeStorage,
+    private readonly input: BeginDatasetMergeInput,
+    private readonly baseVersionId: string | null
+  ) {}
+
+  async writeChanges(
+    changes:
+      | Iterable<MergeChange<DatasetRow, DatasetRow>>
+      | AsyncIterable<MergeChange<DatasetRow, DatasetRow>>
+  ): Promise<void> {
+    this.assertOpen()
+
+    const stagedChanges: MergeChange<DatasetRow, DatasetRow>[] = []
+    for await (const change of changes) {
+      const validationError = getDatasetMergeChangeValidationError(change, this.input.dataset)
+      if (validationError) {
+        throw new LakeStorageError(`[LakeStorage] ${validationError}`)
+      }
+      stagedChanges.push(cloneDatasetMergeChange(change))
+    }
+    this.changes.push(...stagedChanges)
+  }
+
+  async commit(input?: CommitDatasetMergeInput): Promise<DatasetMergeCommitResult> {
+    this.assertOpen()
+    this.closed = true
+    return this.storage.commitMerge({
+      merge: this.input,
+      baseVersionId: this.baseVersionId,
+      changes: this.changes,
+      commit: input,
+    })
+  }
+
+  async abort(): Promise<void> {
+    this.closed = true
+    this.changes.length = 0
+  }
+
+  private assertOpen(): void {
+    if (this.closed) {
+      throw new LakeStorageError("[LakeStorage] Merge session is already closed")
+    }
+  }
+}
+
 export class InMemoryLakeStorage implements LakeStorage {
   private readonly datasets = new Map<string, DatasetDefinition>()
   private readonly versionsByDataset = new Map<string, DatasetVersion[]>()
   private readonly rowsByVersionId = new Map<string, readonly DatasetRow[]>()
   private readonly latestVersionIdByDataset = new Map<string, string>()
+  private readonly commitLocks = new Map<string, Promise<void>>()
 
   async createDataset(definition: DatasetDefinition): Promise<DatasetDefinition> {
     assertDatasetId(definition.id)
@@ -220,6 +302,36 @@ export class InMemoryLakeStorage implements LakeStorage {
     })
   }
 
+  async beginMerge(input: BeginDatasetMergeInput): Promise<LakeMergeSession> {
+    const definition = await this.getDataset(input.dataset.id)
+    if (!definition) {
+      throw new LakeStorageError(`[LakeStorage] Unknown dataset '${input.dataset.id}'`)
+    }
+    if (getDatasetPrimaryKeyColumns(definition) === null) {
+      throw new LakeStorageError(
+        `[LakeStorage] Dataset '${definition.id}' must define a primaryKey before it can be merged.`
+      )
+    }
+
+    const latestVersion = await this.getLatestVersion(definition.id)
+    if (
+      input.expectedLatestVersionId !== undefined &&
+      latestVersion?.versionId !== input.expectedLatestVersionId
+    ) {
+      throw new LakeStorageError(
+        `[LakeStorage] Optimistic merge start failed for dataset '${definition.id}': expected latest version '${input.expectedLatestVersionId}', found '${latestVersion?.versionId ?? "none"}'`
+      )
+    }
+    return new InMemoryLakeMergeSession(
+      this,
+      {
+        ...input,
+        dataset: definition,
+      },
+      latestVersion?.versionId ?? null
+    )
+  }
+
   async getLatestVersion(datasetId: string): Promise<DatasetVersion | null> {
     const latestVersionId = this.latestVersionIdByDataset.get(datasetId)
     if (!latestVersionId) {
@@ -267,6 +379,90 @@ export class InMemoryLakeStorage implements LakeStorage {
     write: BeginDatasetWriteInput
     rows: readonly DatasetRow[]
     commit?: CommitDatasetWriteInput
+  }): Promise<DatasetWriteCommitResult> {
+    return this.withDatasetCommitLock(options.write.dataset.id, () =>
+      this.commitWriteUnlocked(options)
+    )
+  }
+
+  async commitMerge(options: {
+    merge: BeginDatasetMergeInput
+    baseVersionId: string | null
+    changes: readonly MergeChange<DatasetRow, DatasetRow>[]
+    commit?: CommitDatasetMergeInput
+  }): Promise<DatasetMergeCommitResult> {
+    return this.withDatasetCommitLock(options.merge.dataset.id, async () => {
+      const definition = this.datasets.get(options.merge.dataset.id)
+      if (!definition) {
+        throw new LakeStorageError(`[LakeStorage] Unknown dataset '${options.merge.dataset.id}'`)
+      }
+
+      const latestVersion = await this.getLatestVersion(options.merge.dataset.id)
+      const actualVersionId = latestVersion?.versionId ?? null
+      if (actualVersionId !== options.baseVersionId) {
+        throw new LakeStorageError(
+          `[LakeStorage] Optimistic merge commit failed for dataset '${options.merge.dataset.id}': expected latest version '${options.baseVersionId ?? "none"}', found '${actualVersionId ?? "none"}'`
+        )
+      }
+
+      const previousRows = this.rowsByVersionId.get(latestVersion?.versionId ?? "") ?? []
+      const previousByKey = this.rowsByPrimaryKey(previousRows, definition)
+      const finalChanges = new Map<string, MergeChange<DatasetRow, DatasetRow>>()
+      for (const change of options.changes) {
+        const value = change.kind === "upsert" ? change.row : change.key
+        finalChanges.set(encodeDatasetPrimaryKey(definition, value), change)
+      }
+
+      const nextByKey = new Map(previousByKey)
+      for (const [primaryKey, change] of finalChanges) {
+        if (change.kind === "upsert") {
+          nextByKey.set(primaryKey, cloneRow(change.row))
+        } else {
+          nextByKey.delete(primaryKey)
+        }
+      }
+
+      if (this.sameKeyedRowContent(previousByKey, nextByKey, definition.schema)) {
+        return {
+          outcome: "unchanged",
+          version: latestVersion ? cloneDatasetVersion(latestVersion) : null,
+        }
+      }
+
+      const visibleRows = [...nextByKey.values()]
+      const versionId = `ver_${randomUUID()}`
+      const version: DatasetVersion = {
+        datasetId: options.merge.dataset.id,
+        versionId,
+        parentVersionId: latestVersion?.versionId,
+        mode: "merge",
+        createdAt: new Date(),
+        schema: definition.schema,
+        producer: options.merge.producer ? structuredClone(options.merge.producer) : undefined,
+        inputs: options.merge.inputs ? structuredClone(options.merge.inputs) : undefined,
+        rowCount: visibleRows.length,
+        sizeBytes: new TextEncoder().encode(
+          visibleRows.map((row) => JSON.stringify(row)).join("\n")
+        ).byteLength,
+      }
+
+      const versions = this.versionsByDataset.get(options.merge.dataset.id) ?? []
+      versions.push(version)
+      this.versionsByDataset.set(options.merge.dataset.id, versions)
+      this.rowsByVersionId.set(
+        versionId,
+        visibleRows.map((row) => cloneRow(row))
+      )
+      this.latestVersionIdByDataset.set(options.merge.dataset.id, versionId)
+
+      return { outcome: "created", version: cloneDatasetVersion(version) }
+    })
+  }
+
+  private async commitWriteUnlocked(options: {
+    write: BeginDatasetWriteInput
+    rows: readonly DatasetRow[]
+    commit?: CommitDatasetWriteInput
   }): Promise<DatasetWriteCommitResult> {
     const definition = this.datasets.get(options.write.dataset.id)
     if (!definition) {
@@ -285,6 +481,8 @@ export class InMemoryLakeStorage implements LakeStorage {
       }
     }
 
+    this.assertKeyedWriteIsUnique(mode, options.rows, latestVersion, definition)
+
     // Content-identical snapshots and empty appends reuse the latest version
     // instead of creating a new one.
     if (
@@ -345,4 +543,88 @@ export class InMemoryLakeStorage implements LakeStorage {
       previousRows.map((row) => rowContentKey(row, schema))
     )
   }
+
+  private assertKeyedWriteIsUnique(
+    mode: DatasetWriteMode,
+    rows: readonly DatasetRow[],
+    latestVersion: DatasetVersion | null,
+    definition: DatasetDefinition
+  ): void {
+    if (getDatasetPrimaryKeyColumns(definition) === null) {
+      return
+    }
+
+    const stagedByKey = this.rowsByPrimaryKey(rows, definition)
+    if (mode !== "append" || latestVersion === null) {
+      return
+    }
+
+    const previousRows = this.rowsByVersionId.get(latestVersion.versionId) ?? []
+    const previousByKey = this.rowsByPrimaryKey(previousRows, definition)
+    for (const primaryKey of stagedByKey.keys()) {
+      if (previousByKey.has(primaryKey)) {
+        throw new LakeStorageError(
+          `[LakeStorage] Dataset '${definition.id}' append contains duplicate primary key ${primaryKey}.`
+        )
+      }
+    }
+  }
+
+  private rowsByPrimaryKey(
+    rows: readonly DatasetRow[],
+    definition: DatasetDefinition
+  ): Map<string, DatasetRow> {
+    const byKey = new Map<string, DatasetRow>()
+    for (const row of rows) {
+      const primaryKey = encodeDatasetPrimaryKey(definition, row)
+      if (byKey.has(primaryKey)) {
+        throw new LakeStorageError(
+          `[LakeStorage] Dataset '${definition.id}' contains duplicate primary key ${primaryKey}.`
+        )
+      }
+      byKey.set(primaryKey, row)
+    }
+    return byKey
+  }
+
+  private sameKeyedRowContent(
+    left: ReadonlyMap<string, DatasetRow>,
+    right: ReadonlyMap<string, DatasetRow>,
+    schema: DatasetSchema
+  ): boolean {
+    if (left.size !== right.size) {
+      return false
+    }
+    for (const [primaryKey, leftRow] of left) {
+      const rightRow = right.get(primaryKey)
+      if (
+        rightRow === undefined ||
+        rowContentKey(leftRow, schema) !== rowContentKey(rightRow, schema)
+      ) {
+        return false
+      }
+    }
+    return true
+  }
+
+  private async withDatasetCommitLock<T>(datasetId: string, run: () => Promise<T>): Promise<T> {
+    const previous = this.commitLocks.get(datasetId) ?? Promise.resolve()
+    const ready = previous.catch(() => {})
+    let release!: () => void
+    const next = new Promise<void>((resolve) => {
+      release = resolve
+    })
+    const current = ready.then(() => next)
+    this.commitLocks.set(datasetId, current)
+
+    await ready
+    try {
+      return await run()
+    } finally {
+      release()
+      if (this.commitLocks.get(datasetId) === current) {
+        this.commitLocks.delete(datasetId)
+      }
+    }
+  }
 }

package/src/lake-storage/index.ts

@@ -2,7 +2,9 @@ export type {
   DatasetColumnDefinition,
   DatasetColumnType,
   DatasetDefinition,
+  DatasetPrimaryKey,
   DatasetSchema,
+  MergeChange,
 } from "../datasets"
 export type { JsonValue } from "../json"
 export type {
@@ -17,6 +19,18 @@ export {
 } from "./definition-updates"
 export { LakeStorageError } from "./errors"
 export { InMemoryLakeStorage } from "./in-memory"
+export type {
+  BeginDatasetMergeInput,
+  CommitDatasetMergeInput,
+  DatasetMergeCommitResult,
+  LakeMergeSession,
+} from "./merge"
+export {
+  cloneDatasetMergeChange,
+  encodeDatasetPrimaryKey,
+  getDatasetMergeChangeValidationError,
+  getDatasetPrimaryKeyColumns,
+} from "./merge-validation"
 export type {
   ExecuteSqlTransformInput,
   LakeSqlExecutor,