@defrex/autobuild 0.8.0 → 0.10.0

package/packages/core/src/store/local/store.ts

@@ -11,7 +11,7 @@
  * content-addressed.
  */
 import { Database } from 'bun:sqlite'
-import { and, asc, desc, eq, gt, inArray, sql } from 'drizzle-orm'
+import { and, asc, desc, eq, gt, gte, inArray, or, sql } from 'drizzle-orm'
 import { drizzle, type BunSQLiteDatabase } from 'drizzle-orm/bun-sqlite'
 import { mkdirSync } from 'node:fs'
 import { access, copyFile, mkdtemp, rm } from 'node:fs/promises'
@@ -40,7 +40,14 @@ import {
   type SessionEventWrite,
 } from '../../events/sessions'
 import { createBuildScopedStore } from '../build-scope'
+import type { ReducerSnapshot, SnapshotScope } from '../snapshots'
+import { validateCreatedEvent } from '../new-build'
 import { createSessionScopedStore } from '../session-handle'
+import { DIGEST_EVENT_TYPES, type DigestEventRow, reduceBuildDigest } from '../digest'
+import {
+  REPOSITORY_STATE_EVENT_TYPES,
+  readRepoStateEventsWithAnchorRecheck,
+} from '../repo-state-events'
 import {
   DEFAULT_ARTIFACT_RETENTION_MAX_REVISIONS,
   isRetentionManagedKind,
@@ -49,7 +56,7 @@ import {
 import { pollingSubscribe } from '../subscribe'
 import { StreamLocks } from '../streams/lock'
 import { assembleUIMessageDocument } from '../streams/assemble'
-import { readEventsWithWait, readStreamWithWait } from '../streams/wait'
+import { EVENT_WAIT_POLL_MS, readEventsWithWait, readStreamWithWait } from '../streams/wait'
 import {
   serializedBatchSize,
   STREAM_BATCH_MAX_BYTES,
@@ -76,6 +83,7 @@ import {
   type ArtifactMeta,
   type BlobStore,
   type BuildRecord,
+  type BuildDigest,
   type BuildScopedStore,
   type BuildStore,
   type Clock,
@@ -91,6 +99,19 @@ import {
   type SubscribeOptions,
   type Unsubscribe,
 } from '../types'
+import {
+  DEFAULT_TICKET_ASSET_LIMITS,
+  loadTicketAsset,
+  storeTicketAssetBlobs,
+  resolvePinnedAsset,
+  summarizeTicketAsset,
+  validateTicketAssetInput,
+  type TicketAsset,
+  type TicketAssetInput,
+  type TicketAssetLimits,
+  type TicketAssetMeta,
+  type TicketAssetSummary,
+} from '../ticket-assets'
 import { DirBlobStore } from './blobs'
 import {
   artifacts,
@@ -102,6 +123,7 @@ import {
   sessionArtifacts,
   sessionEvents,
   sessions,
+  ticketAssets,
   streamChunks,
   streams,
 } from './schema'
@@ -111,7 +133,21 @@ import {
  * the drizzle schema is the source of truth; this is its inlined form so
  * opening a store never needs a migration step.
  */
-const BOOTSTRAP_DDL = [
+// Exported for the schema-parity test, which pins this DDL against the
+// drizzle schema; schema.ts stays the source of truth, this is its inlined form.
+export const BOOTSTRAP_DDL = [
+  // Reducer snapshots: additive and idempotent, so databases that predate the
+  // table gain it on open.
+  `CREATE TABLE IF NOT EXISTS reducer_snapshots (
+    scope_kind TEXT NOT NULL,
+    scope_key TEXT NOT NULL,
+    reducer TEXT NOT NULL,
+    version INTEGER NOT NULL,
+    cursor INTEGER NOT NULL,
+    state TEXT NOT NULL,
+    updated_at TEXT NOT NULL,
+    PRIMARY KEY (scope_kind, scope_key, reducer)
+  )`,
   `CREATE TABLE IF NOT EXISTS builds (
     slug TEXT PRIMARY KEY,
     repo TEXT NOT NULL,
@@ -134,6 +170,10 @@ const BOOTSTRAP_DDL = [
     payload TEXT NOT NULL,
     PRIMARY KEY (build, seq)
   )`,
+  // The build-digest scan (AUT-487): type-leading so the digest query's cost
+  // grows with the observation/terminal events themselves, not with total
+  // history. Idempotent at open, so pre-existing databases gain it too.
+  `CREATE INDEX IF NOT EXISTS events_type_build_seq ON events (type, build, seq)`,
   `CREATE TABLE IF NOT EXISTS artifacts (
     build TEXT NOT NULL,
     kind TEXT NOT NULL,
@@ -170,6 +210,19 @@ const BOOTSTRAP_DDL = [
     created_at TEXT NOT NULL,
     PRIMARY KEY (repo, kind, revision)
   )`,
+  `CREATE TABLE IF NOT EXISTS ticket_assets (
+    repo TEXT NOT NULL,
+    ticket_id TEXT NOT NULL,
+    kind TEXT NOT NULL,
+    name TEXT NOT NULL,
+    revision INTEGER NOT NULL,
+    layout TEXT NOT NULL,
+    manifest TEXT NOT NULL,
+    size INTEGER NOT NULL,
+    removed INTEGER NOT NULL,
+    created_at TEXT NOT NULL,
+    PRIMARY KEY (repo, ticket_id, kind, name, revision)
+  )`,
   `CREATE TABLE IF NOT EXISTS sessions (
     id TEXT PRIMARY KEY,
     repo TEXT NOT NULL,
@@ -248,6 +301,15 @@ export interface SqliteBuildStoreOptions {
   retention?: { maxRevisions?: number }
 }
 
+function ticketAssetKey(repo: string, ticketId: string, kind: string, name: string) {
+  return and(
+    eq(ticketAssets.repo, repo),
+    eq(ticketAssets.ticketId, ticketId),
+    eq(ticketAssets.kind, kind),
+    eq(ticketAssets.name, name),
+  )
+}
+
 export class SqliteBuildStore implements BuildStore {
   private readonly sqlite: Database
   private readonly db: BunSQLiteDatabase
@@ -399,6 +461,7 @@ export class SqliteBuildStore implements BuildStore {
   }
 
   async createBuild(input: NewBuildInput): Promise<BuildRecord> {
+    const created = validateCreatedEvent(input)
     const ts = this.now()
     return this.writeTx(() => {
       if (this.buildRow(input.slug)) {
@@ -416,6 +479,7 @@ export class SqliteBuildStore implements BuildStore {
           updatedAt: ts,
         })
         .run()
+      if (created !== undefined) this.appendInTx(input.slug, created)
       return this.toRecord(this.requireBuild(input.slug))
     })
   }
@@ -430,6 +494,59 @@ export class SqliteBuildStore implements BuildStore {
     return rows.map((row) => this.toRecord(row))
   }
 
+  async getRepoBuildDigests(repo: string): Promise<Map<string, BuildDigest>> {
+    // From the builds side (AUT-487): the type filter lives in the ON clause
+    // so a build whose log holds none of the digest-relevant event
+    // types still yields its row — a join that starts from `events` would
+    // silently drop such builds and break the operation's completeness
+    // contract. Only the digest-relevant types are fetched, then one shared
+    // derivation
+    // per build, so the answer cannot drift from `reduceBuild`.
+    const rows = this.sqlite
+      .query(
+        `SELECT b.slug AS slug, e.seq AS seq, e.ts AS ts, e.type AS type,
+                CASE WHEN e.type = 'pr-attachment.reclaimed'
+                  THEN json_extract(e.payload, '$.hostedSeq') END AS hostedSeq,
+                CASE WHEN e.type IN ('execution.started', 'execution.ended')
+                  THEN json_extract(e.payload, '$.instance') END AS instance
+         FROM builds b
+         LEFT JOIN events e
+           ON e.build = b.slug AND e.type IN (${DIGEST_EVENT_TYPES.map(() => '?').join(', ')})
+         WHERE b.repo = ?
+         ORDER BY b.slug, e.seq`,
+      )
+      .all(...DIGEST_EVENT_TYPES, repo) as {
+      slug: string
+      seq: number | null
+      ts: string | null
+      type: string | null
+      hostedSeq: number | null
+      instance: string | null
+    }[]
+    const eventsByBuild = new Map<string, DigestEventRow[]>()
+    for (const row of rows) {
+      if (row.seq === null || row.ts === null || row.type === null) continue
+      const events = eventsByBuild.get(row.slug) ?? []
+      events.push({
+        type: row.type as AbEvent['type'],
+        seq: row.seq,
+        ts: row.ts,
+        ...(row.hostedSeq !== null ? { hostedSeq: row.hostedSeq } : {}),
+        ...(row.instance !== null ? { instance: row.instance } : {}),
+      })
+      eventsByBuild.set(row.slug, events)
+    }
+    const digests = new Map<string, BuildDigest>()
+    for (const row of rows) {
+      if (digests.has(row.slug)) continue
+      digests.set(row.slug, {
+        slug: row.slug,
+        ...reduceBuildDigest(eventsByBuild.get(row.slug) ?? []),
+      })
+    }
+    return digests
+  }
+
   /** Current build-stream tail inside an open transaction. */
   private currentSeqInTx(slug: string): number {
     this.requireBuild(slug)
@@ -664,6 +781,10 @@ export class SqliteBuildStore implements BuildStore {
       },
       waitSeconds: opts?.waitSeconds,
       signal: opts?.signal,
+      // Hosted budget: held build-event reads poll at most once per second
+      // (AUT-394) — not the 25 ms stream default, which is presentation
+      // content's cadence.
+      pollMs: EVENT_WAIT_POLL_MS,
     })
   }
 
@@ -935,9 +1056,163 @@ export class SqliteBuildStore implements BuildStore {
       },
       waitSeconds: opts?.waitSeconds,
       signal: opts?.signal,
+      // Hosted budget: held repository-event reads poll at most once per
+      // second (AUT-394) — not the 25 ms stream default, which is
+      // presentation content's cadence.
+      pollMs: EVENT_WAIT_POLL_MS,
     })
   }
 
+  async getRepoHighWater(repo: string): Promise<number> {
+    this.requireRepo(repo)
+    const row = this.db
+      .select({ seq: sql<number | null>`max(${repoEvents.seq})` })
+      .from(repoEvents)
+      .where(eq(repoEvents.repo, repo))
+      .get()
+    return row?.seq ?? 0
+  }
+
+  /** The scope's log tail, or `undefined` for a scope that does not exist. */
+  private snapshotTail(scope: SnapshotScope): number | undefined {
+    if (scope.kind === 'build') {
+      const row = this.sqlite
+        .query(
+          'SELECT (SELECT COALESCE(MAX(seq), 0) FROM events WHERE build = ?1) AS tail FROM builds WHERE slug = ?1',
+        )
+        .get(scope.slug) as { tail: number } | null
+      return row?.tail
+    }
+    const row = this.sqlite
+      .query(
+        'SELECT (SELECT COALESCE(MAX(seq), 0) FROM repo_events WHERE repo = ?1) AS tail FROM repo_streams WHERE repo = ?1',
+      )
+      .get(scope.repo) as { tail: number } | null
+    return row?.tail
+  }
+
+  async getReducerSnapshot(
+    scope: SnapshotScope,
+    reducer: string,
+    version: number,
+  ): Promise<ReducerSnapshot | null> {
+    const tail = this.snapshotTail(scope)
+    if (tail === undefined) return null
+    const row = this.sqlite
+      .query(
+        'SELECT version, cursor, state FROM reducer_snapshots WHERE scope_kind = ?1 AND scope_key = ?2 AND reducer = ?3',
+      )
+      .get(scope.kind, scope.kind === 'build' ? scope.slug : scope.repo, reducer) as {
+      version: number
+      cursor: number
+      state: string
+    } | null
+    if (row === null || row.version !== version || row.cursor > tail) return null
+    try {
+      return { version: row.version, cursor: row.cursor, state: JSON.parse(row.state) }
+    } catch {
+      return null
+    }
+  }
+
+  async putReducerSnapshot(
+    scope: SnapshotScope,
+    reducer: string,
+    snapshot: ReducerSnapshot,
+  ): Promise<boolean> {
+    const tail = this.snapshotTail(scope)
+    if (tail === undefined || snapshot.cursor > tail) return false
+    // One statement: the stored cursor never decreases, whatever interleaves.
+    const result = this.sqlite
+      .query(
+        `INSERT INTO reducer_snapshots (scope_kind, scope_key, reducer, version, cursor, state, updated_at)
+         VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)
+         ON CONFLICT (scope_kind, scope_key, reducer) DO UPDATE SET
+           version = excluded.version, cursor = excluded.cursor,
+           state = excluded.state, updated_at = excluded.updated_at
+         WHERE excluded.cursor > reducer_snapshots.cursor
+            OR (excluded.cursor = reducer_snapshots.cursor AND excluded.version > reducer_snapshots.version)`,
+      )
+      .run(
+        scope.kind,
+        scope.kind === 'build' ? scope.slug : scope.repo,
+        reducer,
+        snapshot.version,
+        snapshot.cursor,
+        JSON.stringify(snapshot.state ?? null),
+        this.clock().toISOString(),
+      )
+    return result.changes > 0
+  }
+
+  async getRepoStateEvents(repo: string): Promise<RepositoryEvent[]> {
+    this.requireRepo(repo)
+    // Two queries, mirroring the oracle's derivation: the latest
+    // run-started anchor, then one select of durable types plus the tail
+    // from that anchor. No new index: local journals are small and the
+    // existing (repo, seq) PK index keeps the anchor scan acceptable.
+    //
+    // The two drizzle reads are synchronous, so there is no in-process await
+    // point between them — but a concurrent writer from another process can
+    // still land between the two statements in real time (AUT-551): a
+    // first-ever `dispatcher.run-started` appended between them would be
+    // invisible to the probe, absent from the durable-only selection, yet
+    // present in the journal by the time this method returns. The shared
+    // helper below closes that window with a one-shot anchor re-check on the
+    // durable-only outcome: a durable-only result is returned only when the
+    // journal was anchor-free as of the final probe. When the probe *does*
+    // find an anchor, the anchored select's `gte(seq, anchor)` arm is
+    // monotone-inclusive, so any later run-started is picked up by that same
+    // select; an append after the final re-probe is an ordinary post-read
+    // concurrent append, not this race.
+    return readRepoStateEventsWithAnchorRecheck(
+      async () => {
+        // This probe is a LIMIT 1 select, not an aggregate: it returns zero or
+        // one rows, so row absence is the deliberate empty-journal signal — no
+        // MAX() NULL semantics involved (contrast the Postgres adapter's
+        // MAX(seq) probe, which always returns one row and must test the NULL
+        // explicitly). The `anchor === undefined` branch in the select below is
+        // the explicit durable-only path; a null anchor must never reach the
+        // gte(repoEvents.seq, anchor) arm (in either dialect the comparison is
+        // never true, so the durable-only outcome would then survive only by
+        // accident), and the no-anchor case must not degenerate into
+        // gte(repoEvents.seq, 0), which would select the whole journal.
+        const anchorRows = this.db
+          .select({ seq: repoEvents.seq })
+          .from(repoEvents)
+          .where(and(eq(repoEvents.repo, repo), eq(repoEvents.type, 'dispatcher.run-started')))
+          .orderBy(desc(repoEvents.seq))
+          .limit(1)
+          .all()
+        return anchorRows[0]?.seq
+      },
+      async (anchor) => {
+        const durable = inArray(repoEvents.type, [...REPOSITORY_STATE_EVENT_TYPES])
+        const rows = this.db
+          .select()
+          .from(repoEvents)
+          .where(
+            anchor === undefined
+              ? and(eq(repoEvents.repo, repo), durable)
+              : and(eq(repoEvents.repo, repo), or(gte(repoEvents.seq, anchor), durable)),
+          )
+          .orderBy(asc(repoEvents.seq))
+          .all()
+        return rows.map(
+          (row) =>
+            ({
+              repo: row.repo,
+              seq: row.seq,
+              ts: row.ts,
+              actor: row.actor,
+              type: row.type,
+              payload: row.payload,
+            }) as RepositoryEvent,
+        )
+      },
+    )
+  }
+
   async putRepoArtifact(repo: string, artifact: ArtifactInput): Promise<RepositoryArtifactMeta> {
     const prepared = await this.prepareArtifact(artifact)
     return this.writeTx(() => this.depositRepoInTx(repo, prepared))
@@ -994,6 +1269,144 @@ export class SqliteBuildStore implements BuildStore {
       .map((row) => this.toRepoMeta(row))
   }
 
+  async ticketAssetLimits(_repo: string): Promise<TicketAssetLimits> {
+    return { ...DEFAULT_TICKET_ASSET_LIMITS }
+  }
+
+  private toTicketAssetMeta(row: typeof ticketAssets.$inferSelect): TicketAssetMeta {
+    return {
+      repo: row.repo,
+      ticketId: row.ticketId,
+      kind: row.kind,
+      name: row.name,
+      revision: row.revision,
+      layout: row.layout,
+      size: row.size,
+      entries: row.manifest,
+      createdAt: row.createdAt,
+    }
+  }
+
+  async putTicketAsset(
+    repo: string,
+    ticketId: string,
+    asset: TicketAssetInput,
+  ): Promise<TicketAssetMeta> {
+    const size = validateTicketAssetInput(asset)
+    const entries = await storeTicketAssetBlobs(this.blobs, asset.entries)
+    return this.writeTx(() => {
+      const latest = this.db
+        .select({ revision: ticketAssets.revision })
+        .from(ticketAssets)
+        .where(ticketAssetKey(repo, ticketId, asset.kind, asset.name))
+        .orderBy(desc(ticketAssets.revision))
+        .limit(1)
+        .get()
+      const row: typeof ticketAssets.$inferInsert = {
+        repo,
+        ticketId,
+        kind: asset.kind,
+        name: asset.name,
+        revision: (latest?.revision ?? -1) + 1,
+        layout: asset.layout,
+        manifest: entries,
+        size,
+        removed: 0,
+        createdAt: this.clock().toISOString(),
+      }
+      this.db.insert(ticketAssets).values(row).run()
+      return this.toTicketAssetMeta(row as typeof ticketAssets.$inferSelect)
+    })
+  }
+
+  async getTicketAsset(
+    repo: string,
+    ticketId: string,
+    kind: string,
+    name: string,
+    rev?: number,
+  ): Promise<TicketAsset | null> {
+    const scoped = ticketAssetKey(repo, ticketId, kind, name)
+    const row =
+      rev === undefined
+        ? this.db
+            .select()
+            .from(ticketAssets)
+            .where(scoped)
+            .orderBy(desc(ticketAssets.revision))
+            .limit(1)
+            .get()
+        : this.db
+            .select()
+            .from(ticketAssets)
+            .where(and(scoped, eq(ticketAssets.revision, rev)))
+            .get()
+    if (!row || row.removed) return null
+    return loadTicketAsset(this.blobs, this.toTicketAssetMeta(row))
+  }
+
+  async getPinnedTicketAsset(
+    slug: string,
+    kind: string,
+    name: string,
+    rev?: number,
+  ): Promise<TicketAsset | null> {
+    return resolvePinnedAsset(this, slug, kind, name, rev)
+  }
+
+  async listTicketAssets(
+    repo: string,
+    ticketId: string,
+    opts?: { revisions?: boolean },
+  ): Promise<TicketAssetSummary[]> {
+    const rows = this.db
+      .select()
+      .from(ticketAssets)
+      .where(and(eq(ticketAssets.repo, repo), eq(ticketAssets.ticketId, ticketId)))
+      .orderBy(asc(ticketAssets.kind), asc(ticketAssets.name), asc(ticketAssets.revision))
+      .all()
+    if (opts?.revisions) {
+      return rows
+        .filter((row) => !row.removed)
+        .map((row) => summarizeTicketAsset(this.toTicketAssetMeta(row)))
+    }
+    const latest = new Map<string, typeof ticketAssets.$inferSelect>()
+    for (const row of rows) latest.set(JSON.stringify([row.kind, row.name]), row)
+    return [...latest.values()]
+      .filter((row) => !row.removed)
+      .map((row) => summarizeTicketAsset(this.toTicketAssetMeta(row)))
+  }
+
+  async removeTicketAsset(
+    repo: string,
+    ticketId: string,
+    kind: string,
+    name: string,
+  ): Promise<TicketAssetMeta | null> {
+    return this.writeTx(() => {
+      const latest = this.db
+        .select()
+        .from(ticketAssets)
+        .where(ticketAssetKey(repo, ticketId, kind, name))
+        .orderBy(desc(ticketAssets.revision))
+        .limit(1)
+        .get()
+      if (!latest || latest.removed) return null
+      this.db
+        .insert(ticketAssets)
+        .values({
+          ...latest,
+          revision: latest.revision + 1,
+          manifest: [],
+          size: 0,
+          removed: 1,
+          createdAt: this.clock().toISOString(),
+        })
+        .run()
+      return this.toTicketAssetMeta(latest)
+    })
+  }
+
   async claimRepoLease(repo: string, holder: string, ttlMs: number): Promise<boolean> {
     return this.writeTx(() => {
       const row = this.requireRepo(repo)
@@ -1189,6 +1602,26 @@ export class SqliteBuildStore implements BuildStore {
     return this.writeTx(() => this.appendSessionInTx(id, validated) as SessionEventEnvelope<T>)
   }
 
+  async appendSessionEventIfCurrent<T extends SessionEventType>(
+    id: string,
+    expectedSeq: number,
+    event: SessionEventWrite<T>,
+  ): Promise<SessionEventEnvelope<T> | null> {
+    validateExpectedSeq(expectedSeq)
+    const validated = validateSessionEventWrite(event)
+    return this.writeTx(() => {
+      // BEGIN IMMEDIATE serializes this comparison and append across every
+      // SQLite connection/process using the same store file.
+      const row = this.db
+        .select({ count: sql<number>`count(*)` })
+        .from(sessionEvents)
+        .where(eq(sessionEvents.session, id))
+        .get()
+      if ((row?.count ?? 0) !== expectedSeq) return null
+      return this.appendSessionInTx(id, validated) as SessionEventEnvelope<T>
+    })
+  }
+
   async getSessionEvents(
     id: string,
     sinceSeq = 0,
@@ -1214,7 +1647,15 @@ export class SqliteBuildStore implements BuildStore {
           }) as SessionEvent,
       )
     }
-    return readEventsWithWait({ read, waitSeconds: opts?.waitSeconds, signal: opts?.signal })
+    return readEventsWithWait({
+      read,
+      waitSeconds: opts?.waitSeconds,
+      signal: opts?.signal,
+      // Hosted budget: held session-event reads poll at most once per second
+      // (AUT-394), matching the build and repository event reads — not the
+      // 25 ms stream default, which is presentation content's cadence.
+      pollMs: EVENT_WAIT_POLL_MS,
+    })
   }
 
   /** Runs inside an open transaction — see `depositInTx` for `prune`. */