@sixb/sqlite 0.1.7 → 0.1.9

package/src/objects/selected-reader.ts

@@ -0,0 +1,119 @@
+import type { Database } from "bun:sqlite"
+import type {
+  CompiledSelectedObjectReadScope,
+  ObjectQueryCapabilities,
+  ObjectReadExecutionLimits,
+  ObjectReadStorage,
+} from "@sixb/core/storage"
+import {
+  assertObjectReaderProject,
+  assertObjectReadFacetCount,
+  assertObjectReadOutputWithinLimit,
+  ObjectReadLimitExceededError,
+  snapshotObjectReadExecutionLimits,
+} from "@sixb/core/storage"
+import { runDeferredReadTransaction } from "../transactions"
+import {
+  compileSqliteSelectedObjectReadSource,
+  type SqliteSelectedObjectReadSource,
+} from "./read-scope"
+import { assertLinkQueryLimit, SqliteObjectReader } from "./reader"
+
+/** Keep the traversal probe and every terminal statement in the same synchronous snapshot. */
+export function createSqliteSelectedReader(
+  db: Database,
+  params: {
+    projectId: string
+    scope: CompiledSelectedObjectReadScope
+    limits: ObjectReadExecutionLimits
+  },
+  queryCapabilities: () => ObjectQueryCapabilities
+): ObjectReadStorage {
+  const projectId = params.projectId
+  const limits = snapshotObjectReadExecutionLimits(params.limits)
+  const source = compileSqliteSelectedObjectReadSource(
+    projectId,
+    params.scope,
+    limits.maxTraversalFacts
+  )
+  const reader = new SqliteObjectReader(db, source)
+  const assertProject = (actualProjectId: string): void =>
+    assertObjectReaderProject(projectId, actualProjectId)
+  const read = <T>(run: () => T): T =>
+    runDeferredReadTransaction(db, () => {
+      assertTraversalBudget(db, source, limits.maxTraversalFacts)
+      const value = run()
+      assertObjectReadOutputWithinLimit(value, limits)
+      return value
+    })
+  const readMap = <TKey, TValue>(run: () => Map<TKey, TValue>): Map<TKey, TValue> =>
+    runDeferredReadTransaction(db, () => {
+      assertTraversalBudget(db, source, limits.maxTraversalFacts)
+      const value = run()
+      assertObjectReadOutputWithinLimit([...value.entries()], limits)
+      return value
+    })
+
+  return Object.freeze({
+    queryCapabilities,
+    queryObjects: async (input) => {
+      assertProject(input.projectId)
+      return read(() => reader.queryObjects(input))
+    },
+    countObjects: async (input) => {
+      assertProject(input.projectId)
+      return read(() => reader.countObjects(input))
+    },
+    existsObjects: async (input) => {
+      assertProject(input.projectId)
+      return read(() => reader.existsObjects(input))
+    },
+    facetObjects: async (input) => {
+      assertProject(input.projectId)
+      assertObjectReadFacetCount(input.facets.length)
+      return read(() => reader.facetObjects(input))
+    },
+    getByPrimaryId: async (input) => {
+      assertProject(input.projectId)
+      return read(() => reader.getByPrimaryId(input))
+    },
+    selectsObjectProperties: async (input) => {
+      assertProject(input.projectId)
+      return read(() => reader.selectsObjectProperties(input))
+    },
+    listLinks: async (input) => {
+      assertProject(input.projectId)
+      return read(() => reader.listLinks(input))
+    },
+    getByPrimaryIdBatch: async (input) => {
+      assertProject(input.projectId)
+      return readMap(() => reader.getByPrimaryIdBatch(input))
+    },
+    listLinksBatch: async (input) => {
+      assertProject(input.projectId)
+      return readMap(() => reader.listLinksBatch(input))
+    },
+    queryLinks: async (input) => {
+      assertProject(input.projectId)
+      assertLinkQueryLimit(input.limit)
+      return read(() => reader.queryLinks(input))
+    },
+    list: async (input) => {
+      assertProject(input.projectId)
+      return read(() => reader.list(input))
+    },
+  } satisfies ObjectReadStorage)
+}
+
+function assertTraversalBudget(
+  db: Database,
+  source: SqliteSelectedObjectReadSource,
+  maxTraversalFacts: number
+): void {
+  const row = db.query(source.traversalProbe.sql).get(...source.traversalProbe.args) as {
+    total: number | bigint
+  }
+  if (BigInt(row.total) > BigInt(maxTraversalFacts)) {
+    throw new ObjectReadLimitExceededError("traversalFacts", maxTraversalFacts)
+  }
+}

package/src/objects/storage.ts

@@ -0,0 +1,296 @@
+import type { Database } from "bun:sqlite"
+import type {
+  CompiledSelectedObjectReadScope,
+  CountObjectsInput,
+  CountObjectsResult,
+  ExistsObjectsInput,
+  ExistsObjectsResult,
+  FacetObjectsInput,
+  FacetObjectsResult,
+  LinkBatchKey,
+  LinkDirection,
+  ObjectBatchKey,
+  ObjectLinkRow,
+  ObjectQueryCapabilities,
+  ObjectReadExecutionLimits,
+  ObjectReadStorage,
+  ObjectRow,
+  ObjectStorage,
+  QueryObjectLinksInput,
+  QueryObjectLinksResult,
+  QueryObjectsInput,
+  QueryObjectsResult,
+} from "@sixb/core/storage"
+import { installFreshSqliteSchema } from "../migrations"
+import {
+  closeSqliteStoreConnection,
+  openSqliteStoreConnection,
+  type SqliteStoreConnection,
+} from "../transactions"
+import { DEFAULT_OBJECT_QUERY_SOURCE } from "./query-compiler"
+import { assertLinkQueryLimit, SqliteObjectReader } from "./reader"
+import { type DatabaseRow, type LinkDatabaseRow, rowToLink, rowToObject } from "./rows"
+import { createSqliteSelectedReader } from "./selected-reader"
+
+export interface SqliteObjectStorageOptions {
+  /** Path to SQLite database file. Defaults to ':memory:' for in-memory database. */
+  path?: string
+  /** Internal shared connection used by bundled SqliteStorage. */
+  connection?: SqliteStoreConnection
+}
+
+const SQLITE_OBJECT_QUERY_CAPABILITIES: ObjectQueryCapabilities = {
+  queryObjects: true,
+  countObjects: true,
+  existsObjects: true,
+  facetObjects: true,
+  nodes: {
+    start: true,
+    refs: true,
+    filter: true,
+    text: true,
+    sort: true,
+    limit: true,
+    page: true,
+    traverse: true,
+    set: true,
+    project: true,
+    expand: true,
+  },
+  predicateOps: {
+    and: true,
+    or: true,
+    not: true,
+    eq: true,
+    neq: true,
+    lt: true,
+    lte: true,
+    gt: true,
+    gte: true,
+    in: true,
+    exists: true,
+    contains: true,
+  },
+  sortKinds: {
+    property: true,
+  },
+  traversalDirections: {
+    outgoing: true,
+    incoming: true,
+  },
+  setOps: {
+    union: true,
+    intersect: true,
+    subtract: true,
+  },
+  scalarOperations: {
+    string: { equality: true, ordering: true },
+    uuid: { equality: true, ordering: true },
+    boolean: { equality: true },
+    integer: { equality: true, ordering: true },
+    double: { equality: true, ordering: true },
+    decimal: { equality: true },
+    date: { equality: true, ordering: true },
+    timestamp: { equality: true, ordering: true },
+  },
+  limits: {
+    totalCount: true,
+    stablePageTokens: true,
+  },
+  notes: [
+    "SQLite object query pushdown supports start/refs/filter/text/sort/limit/page/traverse/set/project/expand over JSON properties and object links.",
+    "expand hydrates linked objects in-database (top-N per parent via row_number() + json_group_array); core resolves each expansion's cardinality before pushdown, and a mixed/unresolved one stays on the fallback.",
+    "Ordered decimal predicates and sorting use the bounded core fallback because SQLite has no native exact decimal type; canonical decimal equality remains pushdown-safe.",
+    "Relevance sorting, vector search, and unresolved start.includeSubtypes remain planner fallback or rejection cases.",
+  ],
+}
+
+/**
+ * SQLite-based ObjectStorage implementation.
+ *
+ * Stores object projections and links. V1 object-query IR pushdown covers the
+ * scalar JSON-property and link-traversal subset declared by queryCapabilities().
+ */
+export class SqliteObjectStorage implements ObjectStorage {
+  private readonly connection: SqliteStoreConnection
+  private readonly db: Database
+  private readonly reader: SqliteObjectReader
+
+  constructor(options: SqliteObjectStorageOptions = {}) {
+    this.connection = openSqliteStoreConnection(options)
+    this.db = this.connection.db
+    this.reader = new SqliteObjectReader(this.db, DEFAULT_OBJECT_QUERY_SOURCE)
+
+    if (this.connection.installFreshSchema) {
+      installFreshSqliteSchema(this.db)
+    }
+  }
+
+  queryCapabilities(): ObjectQueryCapabilities {
+    return SQLITE_OBJECT_QUERY_CAPABILITIES
+  }
+
+  createSelectedReadScope(params: {
+    projectId: string
+    scope: CompiledSelectedObjectReadScope
+    limits: ObjectReadExecutionLimits
+  }): ObjectReadStorage {
+    return createSqliteSelectedReader(this.db, params, () => this.queryCapabilities())
+  }
+
+  async queryObjects(params: QueryObjectsInput): Promise<QueryObjectsResult> {
+    return this.reader.queryObjects(params)
+  }
+
+  async countObjects(params: CountObjectsInput): Promise<CountObjectsResult> {
+    return this.reader.countObjects(params)
+  }
+
+  async existsObjects(params: ExistsObjectsInput): Promise<ExistsObjectsResult> {
+    return this.reader.existsObjects(params)
+  }
+
+  async facetObjects(params: FacetObjectsInput): Promise<FacetObjectsResult> {
+    return this.reader.facetObjects(params)
+  }
+
+  async getByPrimaryId(params: {
+    projectId: string
+    objectTypeId: string
+    primaryId: string
+  }): Promise<ObjectRow | null> {
+    return this.reader.getByPrimaryId(params)
+  }
+
+  async selectsObjectProperties(
+    params: Parameters<ObjectReadStorage["selectsObjectProperties"]>[0]
+  ): Promise<readonly boolean[]> {
+    return this.reader.selectsObjectProperties(params)
+  }
+
+  async listLinks(params: {
+    projectId: string
+    objectTypeId: string
+    objectId: string
+    linkId?: string
+    direction?: LinkDirection
+  }): Promise<readonly ObjectLinkRow[]> {
+    return this.reader.listLinks(params)
+  }
+
+  async getByPrimaryIdBatch(params: {
+    projectId: string
+    items: readonly { objectTypeId: string; primaryId: string }[]
+  }): Promise<Map<ObjectBatchKey, ObjectRow>> {
+    return this.reader.getByPrimaryIdBatch(params)
+  }
+
+  async listLinksBatch(params: {
+    projectId: string
+    direction?: LinkDirection
+    items: readonly { objectTypeId: string; objectId: string; linkId: string }[]
+  }): Promise<Map<LinkBatchKey, ObjectLinkRow[]>> {
+    return this.reader.listLinksBatch(params)
+  }
+
+  async queryLinks(params: QueryObjectLinksInput): Promise<QueryObjectLinksResult> {
+    assertLinkQueryLimit(params.limit)
+    return this.reader.queryLinks(params)
+  }
+
+  async listIncidentLinksBatch(params: {
+    projectId: string
+    items: readonly { objectTypeId: string; objectId: string }[]
+  }): Promise<readonly ObjectLinkRow[]> {
+    if (params.items.length === 0) return []
+
+    const rows = this.db
+      .query(
+        `
+          WITH requested AS (
+            SELECT
+              json_extract(value, '$.objectTypeId') AS object_type_id,
+              json_extract(value, '$.objectId') AS object_id
+            FROM json_each(?)
+          )
+          SELECT link.*
+          FROM links AS link
+          JOIN requested
+            ON requested.object_type_id = link.source_type_id
+           AND requested.object_id = link.source_id
+          WHERE link.project_id = ?
+          UNION
+          SELECT link.*
+          FROM links AS link
+          JOIN requested
+            ON requested.object_type_id = link.target_type_id
+           AND requested.object_id = link.target_id
+          WHERE link.project_id = ?
+        `
+      )
+      .all(JSON.stringify(params.items), params.projectId, params.projectId) as LinkDatabaseRow[]
+
+    return rows.map((row) => rowToLink(row))
+  }
+
+  async listByPrimaryIdPage(params: {
+    projectId: string
+    objectTypeId: string
+    afterPrimaryId?: string
+    limit: number
+  }): Promise<{ objects: readonly ObjectRow[]; nextPrimaryId?: string }> {
+    assertReconciliationPageLimit(params.limit)
+    const rows = this.db
+      .query(
+        `
+          SELECT * FROM objects
+          WHERE project_id = ? AND object_type_id = ?
+            AND (? IS NULL OR primary_id > ?)
+          ORDER BY primary_id ASC
+          LIMIT ?
+        `
+      )
+      .all(
+        params.projectId,
+        params.objectTypeId,
+        params.afterPrimaryId ?? null,
+        params.afterPrimaryId ?? null,
+        params.limit + 1
+      ) as DatabaseRow[]
+    const hasMore = rows.length > params.limit
+    const objects = rows.slice(0, params.limit).map((row) => rowToObject(row))
+    const last = objects.at(-1)
+    return {
+      objects,
+      ...(hasMore && last ? { nextPrimaryId: last.primaryId } : {}),
+    }
+  }
+
+  async list(params: {
+    projectId: string
+    objectTypeId?: string | readonly string[]
+    primaryIdPrefix?: string
+    primaryIdSuffix?: string
+    updatedAfter?: Date
+    updatedBefore?: Date
+    createdAfter?: Date
+    createdBefore?: Date
+    limit?: number
+    offset?: number
+    orderBy?: "createdAt" | "updatedAt" | "primaryId"
+    order?: "asc" | "desc"
+  }): Promise<{ objects: readonly ObjectRow[]; hasMore: boolean; total: number }> {
+    return this.reader.list(params)
+  }
+
+  /** Close the connection only when this provider owns it. */
+  close(): void {
+    closeSqliteStoreConnection(this.connection)
+  }
+}
+
+function assertReconciliationPageLimit(limit: number): void {
+  if (!Number.isSafeInteger(limit) || limit <= 0) {
+    throw new Error("Object reconciliation page limit must be a positive safe integer.")
+  }
+}

package/src/pipeline-run-storage.ts

@@ -1,6 +1,7 @@
 import type { Database } from "bun:sqlite"
 import { parseSixbFailure, serializeSixbFailure } from "@sixb/core/internal/errors"
 import { assertPipelineRunExecution } from "@sixb/core/internal/pipeline-run-storage-provider"
+import { canRequeuePipelineRunAfterEnqueueFailure } from "@sixb/core/internal/storage"
 import type { DatasetVersionRef } from "@sixb/core/lake-storage"
 import type {
   ExecutionStorage,
@@ -19,11 +20,7 @@ import type {
   StartPipelineRunInput,
   StartPipelineStepRunInput,
 } from "@sixb/core/storage"
-import {
-  canRequeuePipelineRunAfterEnqueueFailure,
-  PIPELINE_RUN_FAILURE_CODES,
-  PipelineRunError,
-} from "@sixb/core/storage"
+import { PIPELINE_RUN_FAILURE_CODES, PipelineRunError } from "@sixb/core/storage"
 import { queryLatestRunsByOwnerId } from "./latest-run-query"
 import { installFreshSqliteSchema } from "./migrations"
 import {

package/src/share-grant-storage.ts

@@ -0,0 +1,252 @@
+import type { Database } from "bun:sqlite"
+import {
+  normalizeGetShareGrantByIdInput,
+  normalizeListShareGrantsInput,
+  normalizeRevokeShareGrantInput,
+  normalizeShareGrantCreate,
+  parseShareGrantRecord,
+} from "@sixb/core/internal/share-grant-storage-provider"
+import type {
+  CreateShareGrantInput,
+  GetShareGrantByIdInput,
+  ListShareGrantsInput,
+  ListShareGrantsResult,
+  RevokeShareGrantInput,
+  ShareGrantRecord,
+  ShareGrantStorage,
+} from "@sixb/core/storage"
+import { ShareGrantStorageError } from "@sixb/core/storage"
+import { installFreshSqliteSchema } from "./migrations"
+import { isUniqueConstraintError } from "./storage-errors"
+import {
+  closeSqliteStoreConnection,
+  openSqliteStoreConnection,
+  type SqliteStoreConnection,
+} from "./transactions"
+
+export interface SqliteShareGrantStorageOptions {
+  readonly path?: string
+  readonly connection?: SqliteStoreConnection
+}
+
+/** SQLite persistence for issued Share grants. Secret handling remains in Core. */
+export class SqliteShareGrantStorage implements ShareGrantStorage {
+  private readonly connection: SqliteStoreConnection
+  private readonly db: Database
+
+  constructor(options: SqliteShareGrantStorageOptions = {}) {
+    this.connection = openSqliteStoreConnection(options)
+    this.db = this.connection.db
+    if (this.connection.installFreshSchema) installFreshSqliteSchema(this.db)
+  }
+
+  async create(input: CreateShareGrantInput): Promise<ShareGrantRecord> {
+    const record = normalizeShareGrantCreate(input)
+    try {
+      this.db
+        .query(
+          `
+            INSERT INTO share_grants (
+              project_id, id, definition_id, target_object_type_id, target_primary_id,
+              issued_by_type, issued_by_id, authority_version, authority_snapshot,
+              authority_digest, token_hash, destination_path, created_at, expires_at
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+          `
+        )
+        .run(
+          record.projectId,
+          record.id,
+          record.definitionId,
+          record.target.objectTypeId,
+          record.target.primaryId,
+          record.issuedBy.type,
+          record.issuedBy.id,
+          record.authoritySnapshot.version,
+          JSON.stringify(record.authoritySnapshot),
+          record.authorityDigest,
+          record.tokenHash,
+          record.destinationPath,
+          record.createdAt.toISOString(),
+          record.expiresAt.toISOString()
+        )
+    } catch (error) {
+      if (isUniqueConstraintError(error)) {
+        throw new ShareGrantStorageError(
+          "duplicate",
+          `[SixbSqlite] Share grant '${record.id}' conflicts with an existing record.`,
+          { cause: error }
+        )
+      }
+      throw error
+    }
+    return this.require(record.projectId, record.id, "create")
+  }
+
+  async getById(input: GetShareGrantByIdInput): Promise<ShareGrantRecord | null> {
+    const normalized = normalizeGetShareGrantByIdInput(input)
+    return this.getRow(normalized.projectId, normalized.id)
+  }
+
+  async list(input: ListShareGrantsInput): Promise<ListShareGrantsResult> {
+    const normalized = normalizeListShareGrantsInput(input)
+    const where = ["project_id = ?"]
+    const args: (string | number)[] = [normalized.projectId]
+    if (normalized.definitionId !== undefined) {
+      where.push("definition_id = ?")
+      args.push(normalized.definitionId)
+    }
+    if (normalized.target !== undefined) {
+      where.push("target_object_type_id = ?", "target_primary_id = ?")
+      args.push(normalized.target.objectTypeId, normalized.target.primaryId)
+    }
+    if (!normalized.includeRevoked) where.push("revoked_at IS NULL")
+    if (!normalized.includeExpired) {
+      where.push("expires_at > ?")
+      args.push(normalized.now.toISOString())
+    }
+
+    const predicate = where.join(" AND ")
+    const totalRow = this.db
+      .query(`SELECT COUNT(*) AS total FROM share_grants WHERE ${predicate}`)
+      .get(...args) as { readonly total: number }
+    const rows = this.db
+      .query(
+        `
+          SELECT * FROM share_grants
+          WHERE ${predicate}
+          ORDER BY created_at DESC, id DESC
+          LIMIT ? OFFSET ?
+        `
+      )
+      .all(...args, normalized.limit, normalized.offset) as SqliteShareGrantRow[]
+    const grants = rows.map(rowToRecord)
+    return {
+      grants,
+      total: totalRow.total,
+      hasMore: normalized.offset + grants.length < totalRow.total,
+    }
+  }
+
+  async revoke(input: RevokeShareGrantInput): Promise<ShareGrantRecord | null> {
+    const normalized = normalizeRevokeShareGrantInput(input)
+    const current = this.getRow(normalized.projectId, normalized.id)
+    if (!current || current.revokedAt) return current
+
+    const revocation = normalizeRevokeShareGrantInput(normalized, current.createdAt)
+    this.db
+      .query(
+        `
+          UPDATE share_grants
+          SET revoked_at = ?, revoked_by_type = ?, revoked_by_id = ?
+          WHERE project_id = ? AND id = ? AND revoked_at IS NULL
+        `
+      )
+      .run(
+        revocation.revokedAt.toISOString(),
+        revocation.revokedBy.type,
+        revocation.revokedBy.id,
+        revocation.projectId,
+        revocation.id
+      )
+    return this.require(revocation.projectId, revocation.id, "revoke")
+  }
+
+  close(): void {
+    closeSqliteStoreConnection(this.connection)
+  }
+
+  private getRow(projectId: string, id: string): ShareGrantRecord | null {
+    const row = this.db
+      .query("SELECT * FROM share_grants WHERE project_id = ? AND id = ?")
+      .get(projectId, id) as SqliteShareGrantRow | null
+    return row ? rowToRecord(row) : null
+  }
+
+  private require(projectId: string, id: string, operation: string): ShareGrantRecord {
+    const record = this.getRow(projectId, id)
+    if (!record) {
+      throw new ShareGrantStorageError(
+        "invalid_record",
+        `[SixbSqlite] Share grant '${id}' disappeared after ${operation}.`
+      )
+    }
+    return record
+  }
+}
+
+interface SqliteShareGrantRow {
+  readonly project_id: unknown
+  readonly id: unknown
+  readonly definition_id: unknown
+  readonly target_object_type_id: unknown
+  readonly target_primary_id: unknown
+  readonly issued_by_type: unknown
+  readonly issued_by_id: unknown
+  readonly authority_version: unknown
+  readonly authority_snapshot: unknown
+  readonly authority_digest: unknown
+  readonly token_hash: unknown
+  readonly destination_path: unknown
+  readonly created_at: unknown
+  readonly expires_at: unknown
+  readonly revoked_at: unknown
+  readonly revoked_by_type: unknown
+  readonly revoked_by_id: unknown
+}
+
+function rowToRecord(row: SqliteShareGrantRow): ShareGrantRecord {
+  const authoritySnapshot = parseStoredJson(row.authority_snapshot, "authority snapshot")
+  if (
+    typeof row.authority_version !== "number" ||
+    !isRecord(authoritySnapshot) ||
+    authoritySnapshot.version !== row.authority_version
+  ) {
+    throw invalidRecord("Share authority version does not match its snapshot.")
+  }
+
+  const hasRevokedAt = row.revoked_at !== null
+  const hasRevokedBy = row.revoked_by_type !== null || row.revoked_by_id !== null
+  if (hasRevokedAt !== hasRevokedBy) {
+    throw invalidRecord("Share revocation time and actor must be stored together.")
+  }
+
+  return parseShareGrantRecord({
+    projectId: row.project_id,
+    id: row.id,
+    definitionId: row.definition_id,
+    target: {
+      objectTypeId: row.target_object_type_id,
+      primaryId: row.target_primary_id,
+    },
+    issuedBy: { type: row.issued_by_type, id: row.issued_by_id },
+    authoritySnapshot,
+    authorityDigest: row.authority_digest,
+    tokenHash: row.token_hash,
+    destinationPath: row.destination_path,
+    createdAt: row.created_at,
+    expiresAt: row.expires_at,
+    ...(hasRevokedAt ? { revokedAt: row.revoked_at } : {}),
+    ...(hasRevokedBy ? { revokedBy: { type: row.revoked_by_type, id: row.revoked_by_id } } : {}),
+  })
+}
+
+function parseStoredJson(value: unknown, field: string): unknown {
+  if (typeof value !== "string") throw invalidRecord(`Share ${field} must be stored as JSON text.`)
+  try {
+    return JSON.parse(value)
+  } catch (cause) {
+    throw new ShareGrantStorageError(
+      "invalid_record",
+      `[SixbSqlite] Share ${field} contains invalid JSON.`,
+      { cause }
+    )
+  }
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value)
+}
+
+function invalidRecord(message: string): ShareGrantStorageError {
+  return new ShareGrantStorageError("invalid_record", `[SixbSqlite] ${message}`)
+}