@crombolo/data 0.14.1 → 0.15.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crombolo/data",
-  "version": "0.14.1",
+  "version": "0.15.0",
   "description": "SQLite and Postgres storage for crombolo: schema, migrations and repositories.",
   "license": "MIT",
   "author": "Crombolo",
@@ -28,6 +28,6 @@
     "access": "public"
   },
   "dependencies": {
-    "@crombolo/core": "0.14.1"
+    "@crombolo/core": "0.15.0"
   }
 }

package/src/index.ts

@@ -33,6 +33,7 @@ export * from './repositories/notifications.ts'
 export * from './repositories/public-access.ts'
 export * from './repositories/redirects.ts'
 export * from './repositories/references.ts'
+export * from './repositories/remote-jobs.ts'
 export * from './repositories/tags.ts'
 export * from './repositories/templates.ts'
 export * from './repositories/transfer-answers.ts'
@@ -67,6 +68,7 @@ export * from './schema/plan.ts'
 export * from './schema/property-searchable.ts'
 export * from './schema/public-access.ts'
 export * from './schema/recycle-bin.ts'
+export * from './schema/remote.ts'
 export * from './schema/rich-text-ui.ts'
 export * from './schema/search-index.ts'
 export * from './schema/seed.ts'

package/src/repositories/remote-jobs.ts

@@ -0,0 +1,191 @@
+/**
+ * Pulls and pushes through the remote protocol (story 107): one row per job,
+ * read by whichever node a client polls, written by the node running it.
+ */
+import type { Db } from '../database.ts'
+import { DbDate } from '../dialect.ts'
+
+export type RemoteJobKind = 'pull' | 'push'
+export type RemoteJobState =
+  | 'queued'
+  | 'running'
+  | 'approval-required'
+  | 'done'
+  | 'failed'
+  | 'refused'
+
+export interface RemoteJob {
+  id: string
+  kind: RemoteJobKind
+  environment: string
+  requestedBy: string
+  requestedByName: string
+  options: Record<string, unknown>
+  state: RemoteJobState
+  stage: string | null
+  progress: number | null
+  fileKey: string | null
+  fileSize: number | null
+  sha256: string | null
+  problem: { code: string; title: string; detail?: string; items?: string[] } | null
+  secret: string | null
+  runId: string | null
+  decidedBy: string | null
+  decidedAt: Date | undefined
+  heartbeatAt: Date
+  expiresAt: Date
+  createdAt: Date
+}
+
+const parse = <T>(value: unknown, fallback: T): T => {
+  if (typeof value !== 'string' || value === '') return fallback
+  try {
+    return JSON.parse(value) as T
+  } catch {
+    return fallback
+  }
+}
+
+function mapJob(row: Record<string, unknown>): RemoteJob {
+  const str = (v: unknown) => (v === null || v === undefined ? null : String(v))
+  return {
+    id: String(row.id),
+    kind: String(row.kind) as RemoteJobKind,
+    environment: String(row.environment),
+    requestedBy: String(row.requested_by),
+    requestedByName: String(row.requested_by_name),
+    options: parse(row.options, {}),
+    state: String(row.state) as RemoteJobState,
+    stage: str(row.stage),
+    progress: row.progress === null || row.progress === undefined ? null : Number(row.progress),
+    fileKey: str(row.file_key),
+    fileSize: row.file_size === null || row.file_size === undefined ? null : Number(row.file_size),
+    sha256: str(row.sha256),
+    problem: parse(row.problem, null),
+    secret: str(row.secret),
+    runId: str(row.run_id),
+    decidedBy: str(row.decided_by),
+    decidedAt: DbDate.fromDb(row.decided_at),
+    heartbeatAt: DbDate.fromDb(row.heartbeat_at) ?? new Date(0),
+    expiresAt: DbDate.fromDb(row.expires_at) ?? new Date(0),
+    createdAt: DbDate.fromDb(row.create_date) ?? new Date(0),
+  }
+}
+
+export type RemoteJobUpdate = Partial<
+  Pick<
+    RemoteJob,
+    | 'state'
+    | 'stage'
+    | 'progress'
+    | 'fileKey'
+    | 'fileSize'
+    | 'sha256'
+    | 'problem'
+    | 'secret'
+    | 'runId'
+    | 'decidedBy'
+    | 'decidedAt'
+    | 'expiresAt'
+  >
+>
+
+const COLUMNS: Record<keyof RemoteJobUpdate, string> = {
+  state: 'state',
+  stage: 'stage',
+  progress: 'progress',
+  fileKey: 'file_key',
+  fileSize: 'file_size',
+  sha256: 'sha256',
+  problem: 'problem',
+  secret: 'secret',
+  runId: 'run_id',
+  decidedBy: 'decided_by',
+  decidedAt: 'decided_at',
+  expiresAt: 'expires_at',
+}
+
+export class RemoteJobRepository {
+  #db: Db
+
+  constructor(db: Db) {
+    this.#db = db
+  }
+
+  async create(job: {
+    id: string
+    kind: RemoteJobKind
+    environment: string
+    requestedBy: string
+    requestedByName: string
+    options: Record<string, unknown>
+    state: RemoteJobState
+    expiresAt: Date
+  }): Promise<void> {
+    const now = DbDate.toDb(new Date())
+    await this.#db.exec(
+      `INSERT INTO remote_job
+         (id, kind, environment, requested_by, requested_by_name, options, state, heartbeat_at, expires_at, create_date)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+      [
+        job.id,
+        job.kind,
+        job.environment,
+        job.requestedBy,
+        job.requestedByName,
+        JSON.stringify(job.options),
+        job.state,
+        now,
+        DbDate.toDb(job.expiresAt),
+        now,
+      ],
+    )
+  }
+
+  async get(id: string): Promise<RemoteJob | undefined> {
+    const rows = await this.#db.query('SELECT * FROM remote_job WHERE id = ?', [id])
+    return rows[0] ? mapJob(rows[0]) : undefined
+  }
+
+  /** Records progress, and that the node running it is still alive. */
+  async update(id: string, change: RemoteJobUpdate): Promise<void> {
+    const sets: string[] = ['heartbeat_at = ?']
+    const values: unknown[] = [DbDate.toDb(new Date())]
+    for (const [field, value] of Object.entries(change) as [keyof RemoteJobUpdate, unknown][]) {
+      sets.push(`${COLUMNS[field]} = ?`)
+      values.push(
+        field === 'problem'
+          ? value === null
+            ? null
+            : JSON.stringify(value)
+          : value instanceof Date
+            ? DbDate.toDb(value)
+            : value,
+      )
+    }
+    await this.#db.exec(`UPDATE remote_job SET ${sets.join(', ')} WHERE id = ?`, [...values, id])
+  }
+
+  /** Moves a job from one state to another only if it is still in the first; true when it moved. */
+  async transition(id: string, from: RemoteJobState, change: RemoteJobUpdate): Promise<boolean> {
+    const rows = await this.#db.query<{ id: string }>(
+      'UPDATE remote_job SET state = ?, heartbeat_at = ? WHERE id = ? AND state = ? RETURNING id',
+      [change.state ?? from, DbDate.toDb(new Date()), id, from],
+    )
+    if (rows.length === 0) return false
+    await this.update(id, change)
+    return true
+  }
+
+  /** Jobs past their expiry, whose files are to be swept. */
+  async expired(now = new Date()): Promise<RemoteJob[]> {
+    const rows = await this.#db.query('SELECT * FROM remote_job WHERE expires_at < ?', [
+      DbDate.toDb(now),
+    ])
+    return rows.map(mapJob)
+  }
+
+  async delete(id: string): Promise<void> {
+    await this.#db.exec('DELETE FROM remote_job WHERE id = ?', [id])
+  }
+}

package/src/repositories/transfer-runs.ts

@@ -138,14 +138,19 @@ export class TransferRunRepository {
   }
 
   /** Runs, newest first. */
-  async recent(limit = 20): Promise<TransferRun[]> {
+  async recent(limit = 20, offset = 0): Promise<TransferRun[]> {
     const rows = await this.#db.query(
-      'SELECT * FROM content_transfer_run ORDER BY started_at DESC, id DESC LIMIT ?',
-      [limit],
+      'SELECT * FROM content_transfer_run ORDER BY started_at DESC, id DESC LIMIT ? OFFSET ?',
+      [limit, offset],
     )
     return rows.map(mapRun)
   }
 
+  async count(): Promise<number> {
+    const rows = await this.#db.query('SELECT COUNT(*) AS n FROM content_transfer_run')
+    return Number(rows[0]?.n ?? 0)
+  }
+
   async changes(runId: string): Promise<TransferChange[]> {
     const rows = await this.#db.query(
       'SELECT * FROM content_transfer_change WHERE run_id = ? ORDER BY id',

package/src/schema/plan.ts

@@ -40,6 +40,7 @@ import { propertySearchableMigration } from './property-searchable.ts'
 import { publicAccessMigration } from './public-access.ts'
 import { recycleBinMigration } from './recycle-bin.ts'
 import { redirectsMigration } from './redirects.ts'
+import { remoteMigration } from './remote.ts'
 import { richTextUiMigration } from './rich-text-ui.ts'
 import { searchIndexMigration } from './search-index.ts'
 import { serverRoleMigration } from './server-role.ts'
@@ -95,4 +96,5 @@ export const cromboloPlan = new MigrationPlan([
   approvalsMigration,
   siteSearchMigration,
   twoFactorMigration,
+  remoteMigration,
 ])

package/src/schema/remote.ts

@@ -0,0 +1,76 @@
+/**
+ * Migration 041 — the remote protocol (story 107). docs/remote-protocol.md.
+ *
+ * `device_authorization` is a sign-in by the OAuth device grant (RFC 8628):
+ * the device code's hash, the code a person types, who approved it, and the
+ * session its tokens belong to — which is what "signed-in devices" lists and
+ * revokes. `remote_job` is a pull or a push: who asked, where it has got to,
+ * the file it made or received in the media store, and the approval a
+ * protected push may wait for — with its passphrase sealed under
+ * `CROMBOLO_SECRET_KEY` meanwhile, since the import cannot run without it. Grants the four remote verbs to the
+ * administrators group.
+ */
+import type { Db } from '../database.ts'
+import type { Migration } from '../migrations.ts'
+import { grantBuiltInPermissions } from './seed.ts'
+import { STATE_TWO_FACTOR } from './two-factor.ts'
+
+export const STATE_REMOTE = '{2b7e5c91-0041-4f3a-8d26-9c4a1e7b3f51}'
+
+export const remoteMigration: Migration = {
+  from: STATE_TWO_FACTOR,
+  to: STATE_REMOTE,
+  name: 'AddRemote',
+  kind: 'expand',
+  release: '0.14.4',
+  async up(db: Db) {
+    const t = db.dialect.types
+    await db.exec(
+      `CREATE TABLE device_authorization (
+         id ${t.identity},
+         device_code_hash ${t.varchar(64)} NOT NULL UNIQUE,
+         user_code ${t.varchar(16)} NOT NULL UNIQUE,
+         client_id ${t.varchar(255)} NOT NULL,
+         client_name ${t.varchar(255)},
+         scope ${t.text} NOT NULL,
+         status ${t.varchar(16)} NOT NULL,
+         user_id ${t.integer} REFERENCES user_account (id),
+         session_id ${t.uuid},
+         interval_seconds ${t.integer} NOT NULL,
+         last_polled_at ${t.timestamp},
+         expires_at ${t.timestamp} NOT NULL,
+         decided_at ${t.timestamp},
+         create_date ${t.timestamp} NOT NULL
+       )`,
+    )
+    await db.exec(
+      'CREATE INDEX ix_device_authorization_session ON device_authorization (session_id)',
+    )
+    await db.exec(
+      `CREATE TABLE remote_job (
+         id ${t.varchar(36)} PRIMARY KEY,
+         kind ${t.varchar(8)} NOT NULL,
+         environment ${t.varchar(64)} NOT NULL,
+         requested_by ${t.varchar(36)} NOT NULL,
+         requested_by_name ${t.varchar(255)} NOT NULL,
+         options ${t.text} NOT NULL,
+         state ${t.varchar(24)} NOT NULL,
+         stage ${t.varchar(64)},
+         progress ${t.integer},
+         file_key ${t.text},
+         file_size ${t.bigint},
+         sha256 ${t.varchar(64)},
+         problem ${t.text},
+         secret ${t.text},
+         run_id ${t.varchar(36)},
+         decided_by ${t.varchar(36)},
+         decided_at ${t.timestamp},
+         heartbeat_at ${t.timestamp} NOT NULL,
+         expires_at ${t.timestamp} NOT NULL,
+         create_date ${t.timestamp} NOT NULL
+       )`,
+    )
+    await db.exec('CREATE INDEX ix_remote_job_state ON remote_job (state)')
+    await grantBuiltInPermissions(db)
+  },
+}

package/src/schema/seed.ts

@@ -4,6 +4,7 @@
  * Password hashing is injected rather than imported so this package stays free of
  * a dependency on @crombolo/auth.
  */
+import { RemotePermissions } from '@crombolo/core'
 import type { Db } from '../database.ts'
 import { DbDate } from '../dialect.ts'
 
@@ -65,6 +66,7 @@ export const BUILT_IN_GROUP_PERMISSIONS: Record<string, readonly string[]> = {
     'Umb.Document.PropertyValue.Write',
     ...FORM_FULL,
     ...ELEMENT_EDIT,
+    ...Object.values(RemotePermissions),
   ],
   editor: [
     'Umb.Document.Create',

package/src/setup.ts

@@ -55,3 +55,17 @@ export function completeSetup(
     }),
   )
 }
+
+/**
+ * Closes setup without creating an administrator: a machine can administer the
+ * site instead (story 034's seeded API client). Answers whether it was pending.
+ */
+export function closeSetup(db: Db): Promise<boolean> {
+  return db.locks.withLock(Locks.KeyValues, () =>
+    db.transaction(async (tx) => {
+      if ((await readSetupState(tx)) !== 'pending') return false
+      await writeSetupState(tx, 'complete')
+      return true
+    }),
+  )
+}