@pikku/kysely 0.13.20 → 0.13.22

package/dist/src/schema/virtual-user-schedule.schema.js

@@ -0,0 +1,43 @@
+/**
+ * The `virtualUserSchedule` table {@link KyselyVirtualUserScheduleStore} writes
+ * to: one row per persona that keeps using the app on its own.
+ *
+ * Its own schema rather than a third table in {@link virtualUserSchema}, and
+ * owned by its own store: recording what a persona did and deciding that it
+ * keeps going on its own are separate opt-ins, and a project that only wants
+ * the first should not be handed a cadence table it never writes to.
+ *
+ * `persona` is the primary key: a persona has one cadence, and a second row for
+ * the same one would mean two copies of the same user acting at once — not a
+ * heavier test, a different and unreproducible one.
+ */
+export const virtualUserScheduleSchema = {
+    name: 'virtual-user-schedule',
+    ownedBy: ['virtualUserScheduleStore'],
+    statements: [
+        (db) => db.schema
+            .createTable('virtualUserSchedule')
+            .addColumn('persona', 'varchar(255)', (col) => col.primaryKey())
+            // 0 or 1 rather than a boolean, matching the run tables: a bare sqlite
+            // driver cannot bind one, and `SerializePlugin` is not installed
+            // everywhere. Off by default, so creating the table spends nothing.
+            .addColumn('enabled', 'integer', (col) => col.defaultTo(0).notNull())
+            .addColumn('disposition', 'varchar(50)', (col) => col.notNull())
+            .addColumn('goals', 'text', (col) => col.defaultTo('[]').notNull())
+            .addColumn('budget', 'text')
+            // The gap to the next run is drawn between these rather than fixed, so
+            // the persona does not keep an appointment.
+            .addColumn('minIntervalMs', 'bigint', (col) => col.notNull())
+            .addColumn('maxIntervalMs', 'bigint', (col) => col.notNull())
+            // The entire schedule. Written before a run is dispatched, so a tick
+            // that dies halfway cannot leave the row due for the next one.
+            .addColumn('nextRunAt', 'timestamp', (col) => col.notNull())
+            .addColumn('lastRunId', 'varchar(36)')
+            .addColumn('lastRunAt', 'timestamp'),
+        // The only read that matters at speed: which rows are due, on every tick.
+        (db) => db.schema
+            .createIndex('idx_virtual_user_schedule_due')
+            .on('virtualUserSchedule')
+            .columns(['enabled', 'nextRunAt']),
+    ],
+};

package/dist/src/schema/virtual-user.schema.d.ts

@@ -1,6 +1,7 @@
 import type { PikkuSchema } from './pikku-schema.types.js';
 /**
- * The `virtualUserRun` table {@link KyselyVirtualUserRunStore} writes to.
+ * The `virtualUserRun` and `virtualUserRunStep` tables
+ * {@link KyselyVirtualUserRunStore} writes to.
  *
  * Deliberately not in `pikkuSchemas`: the runtime never needs this table, only
  * a project that runs virtual users does, and creating it for everyone would

package/dist/src/schema/virtual-user.schema.js

@@ -1,6 +1,7 @@
 import { sql } from 'kysely';
 /**
- * The `virtualUserRun` table {@link KyselyVirtualUserRunStore} writes to.
+ * The `virtualUserRun` and `virtualUserRunStep` tables
+ * {@link KyselyVirtualUserRunStore} writes to.
  *
  * Deliberately not in `pikkuSchemas`: the runtime never needs this table, only
  * a project that runs virtual users does, and creating it for everyone would
@@ -33,6 +34,10 @@ export const virtualUserSchema = {
             .addColumn('goals', 'text', (col) => col.defaultTo('[]').notNull())
             .addColumn('memory', 'text', (col) => col.defaultTo('{}').notNull())
             .addColumn('findings', 'text', (col) => col.defaultTo('[]').notNull())
+            // What the user set out to do and how far each one got. On the run row
+            // rather than beside the steps: there are a handful of them, and every
+            // read of the run wants them.
+            .addColumn('intents', 'text', (col) => col.defaultTo('[]').notNull())
             // Counts the engine kept: steps, calls, mutations, tokens, elapsed.
             .addColumn('tally', 'text')
             .addColumn('stoppedBy', 'varchar(255)')
@@ -51,5 +56,38 @@ export const virtualUserSchema = {
             .createIndex('idx_virtual_user_run_persona')
             .on('virtualUserRun')
             .columns(['persona', 'createdAt']),
+        // One row per turn, kept out of `virtualUserRun` so that listing runs does
+        // not drag a budget's worth of transcript along with it. No foreign key
+        // onto the run for the same reason `persona` is not one: this table is
+        // created by the store at boot on whatever engine the project uses, and a
+        // cascade that half the engines enforce differently is worse than the read
+        // rule that a step with no run is orphaned.
+        (db) => db.schema
+            .createTable('virtualUserRunStep')
+            .addColumn('runId', 'varchar(36)', (col) => col.notNull())
+            // The engine's own step number, which is what the intent records point
+            // at — not a surrogate key, so a transcript reads the same after paging.
+            // Spelled `stepIndex` because `index` is a reserved word in mysql.
+            .addColumn('stepIndex', 'integer', (col) => col.notNull())
+            .addColumn('intentId', 'varchar(255)')
+            // The action as the engine scheduled it, including the `invalid` shape
+            // for a turn the model got wrong — that turn is the interesting one.
+            .addColumn('action', 'text', (col) => col.notNull())
+            .addColumn('status', 'integer')
+            // 0 or 1 rather than a boolean, because a bare sqlite driver cannot
+            // bind one at all and `SerializePlugin` is not installed everywhere.
+            .addColumn('ok', 'integer')
+            // Already truncated by the engine, and stored JSON-encoded rather than
+            // raw: a truncated API response usually starts with a brace, which
+            // `SerializePlugin` would read back as an object rather than the string
+            // the engine actually saw.
+            .addColumn('response', 'text')
+            .addColumn('findingKinds', 'text')
+            .addColumn('tokensIn', 'integer', (col) => col.defaultTo(0).notNull())
+            .addColumn('tokensOut', 'integer', (col) => col.defaultTo(0).notNull())
+            .addPrimaryKeyConstraint('pk_virtual_user_run_step', [
+            'runId',
+            'stepIndex',
+        ]),
     ],
 };

package/dist/tsconfig.tsbuildinfo

@@ -1 +1 @@
-{"root":["../src/coercion-plugin.ts","../src/create-audited-kysely.ts","../src/index.ts","../src/kysely-agent-run-scores.ts","../src/kysely-agent-run-service.ts","../src/kysely-agent-run-state-service.ts","../src/kysely-agent-storage-service.ts","../src/kysely-audit-service.ts","../src/kysely-channel-store.ts","../src/kysely-credential-service.ts","../src/kysely-deployment-service.ts","../src/kysely-eventhub-store.ts","../src/kysely-json.ts","../src/kysely-scope-service.ts","../src/kysely-secret-service.ts","../src/kysely-session-store.ts","../src/kysely-tables.ts","../src/kysely-virtual-user-run-store.ts","../src/kysely-webhook-service.ts","../src/kysely-workflow-mirror.ts","../src/kysely-workflow-run-service.ts","../src/kysely-workflow-service.ts","../src/serialize-plugin.ts","../src/schema/agent.schema.ts","../src/schema/audit.schema.ts","../src/schema/channel.schema.ts","../src/schema/credential.schema.ts","../src/schema/deployment.schema.ts","../src/schema/index.ts","../src/schema/pikku-schema.types.ts","../src/schema/scope.schema.ts","../src/schema/secret.schema.ts","../src/schema/session.schema.ts","../src/schema/virtual-user.schema.ts","../src/schema/webhook.schema.ts","../src/schema/workflow.schema.ts","../bin/pikku-kysely-pure.ts"],"version":"6.0.3"}
+{"root":["../src/coercion-plugin.ts","../src/create-audited-kysely.ts","../src/index.ts","../src/kysely-agent-run-scores.ts","../src/kysely-agent-run-service.ts","../src/kysely-agent-run-state-service.ts","../src/kysely-agent-storage-service.ts","../src/kysely-audit-service.ts","../src/kysely-channel-store.ts","../src/kysely-credential-service.ts","../src/kysely-deployment-service.ts","../src/kysely-eventhub-store.ts","../src/kysely-json.ts","../src/kysely-scope-service.ts","../src/kysely-secret-service.ts","../src/kysely-session-store.ts","../src/kysely-tables.ts","../src/kysely-virtual-user-run-store.ts","../src/kysely-virtual-user-schedule-store.ts","../src/kysely-webhook-service.ts","../src/kysely-workflow-mirror.ts","../src/kysely-workflow-run-service.ts","../src/kysely-workflow-service.ts","../src/serialize-plugin.ts","../src/schema/agent.schema.ts","../src/schema/audit.schema.ts","../src/schema/channel.schema.ts","../src/schema/credential.schema.ts","../src/schema/deployment.schema.ts","../src/schema/index.ts","../src/schema/pikku-schema.types.ts","../src/schema/scope.schema.ts","../src/schema/secret.schema.ts","../src/schema/session.schema.ts","../src/schema/virtual-user-schedule.schema.ts","../src/schema/virtual-user.schema.ts","../src/schema/webhook.schema.ts","../src/schema/workflow.schema.ts","../bin/pikku-kysely-pure.ts"],"version":"6.0.3"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pikku/kysely",
-  "version": "0.13.20",
+  "version": "0.13.22",
   "description": "Kysely-backed Pikku services — workflow state, secrets, sessions, scopes and audit",
   "author": "yasser.fadl@gmail.com",
   "license": "MIT",
@@ -21,7 +21,7 @@
     "prepublishOnly": "yarn build"
   },
   "peerDependencies": {
-    "@pikku/core": "^0.12.89"
+    "@pikku/core": "^0.12.93"
   },
   "dependencies": {
     "kysely": "^0.29.5"

package/src/index.ts

@@ -18,6 +18,7 @@ export {
 } from './create-audited-kysely.js'
 export { KyselyAuditService } from './kysely-audit-service.js'
 export { KyselyVirtualUserRunStore } from './kysely-virtual-user-run-store.js'
+export { KyselyVirtualUserScheduleStore } from './kysely-virtual-user-schedule-store.js'
 
 export {
   SerializePlugin,
@@ -61,4 +62,6 @@ export type { AgentRunService, AgentRunRow } from '@pikku/core/agent'
 export type {
   VirtualUserRunStore,
   VirtualUserRunRecord,
+  VirtualUserScheduleStore,
+  VirtualUserScheduleRecord,
 } from '@pikku/core/virtual-user'

package/src/kysely-secret-service.ts

@@ -121,7 +121,7 @@ export class KyselySecretService implements SecretService {
       .where('key', '=', key)
       .executeTakeFirst()
 
-    if (!row) throw new Error('Requested secret not found')
+    if (!row) throw new Error(`Requested secret not found: ${key}`)
 
     const kek = await this.getKEK(row.keyVersion)
     const result = await envelopeDecrypt<T>(kek, row.ciphertext, row.wrappedDek)

package/src/kysely-tables.ts

@@ -311,6 +311,7 @@ export interface VirtualUserRunTable {
   goals: Generated<string>
   memory: Generated<string>
   findings: Generated<string>
+  intents: Generated<string>
   tally: string | null
   stoppedBy: string | null
   error: string | null
@@ -352,6 +353,51 @@ export interface AuditTable {
   data: string | null
 }
 
+/**
+ * One turn of a run's transcript.
+ *
+ * Its own table rather than a column on {@link VirtualUserRunTable}: a run at a
+ * 500-step budget carries more transcript than every other column together, and
+ * the run list would pay for it on every row.
+ */
+export interface VirtualUserRunStepTable {
+  runId: string
+  /** The engine's step number. `stepIndex` because `index` is reserved in mysql. */
+  stepIndex: number
+  intentId: string | null
+  /** JSON: the action the engine scheduled, or the `invalid` shape. */
+  action: string
+  status: number | null
+  /** 0 or 1: not every driver in this package can bind a boolean. */
+  ok: number | null
+  /** JSON-encoded, so a brace-leading response is not read back as an object. */
+  response: string | null
+  /** JSON array, null when the turn produced no finding. */
+  findingKinds: string | null
+  tokensIn: Generated<number>
+  tokensOut: Generated<number>
+}
+
+/**
+ * One persona's cadence. Nullable timestamps and 0/1 flags for the same driver
+ * reasons as {@link VirtualUserRunTable}.
+ */
+export interface VirtualUserScheduleTable {
+  persona: string
+  /** 0 or 1: not every driver in this package can bind a boolean. */
+  enabled: Generated<number>
+  disposition: string
+  goals: Generated<string>
+  /** JSON, or null for the engine's own default budget. */
+  budget: string | null
+  /** BIGINT, so a driver may hand either back. */
+  minIntervalMs: string | number | bigint
+  maxIntervalMs: string | number | bigint
+  nextRunAt: Date | string
+  lastRunId: string | null
+  lastRunAt: Date | string | null
+}
+
 export interface KyselyPikkuDB {
   pikkuScopes: PikkuScopesTable
   pikkuRoles: PikkuRolesTable
@@ -382,5 +428,7 @@ export interface KyselyPikkuDB {
   webhookDelivery: WebhookDeliveryTable
   webhookDeliveryAttempt: WebhookDeliveryAttemptTable
   virtualUserRun: VirtualUserRunTable
+  virtualUserRunStep: VirtualUserRunStepTable
+  virtualUserSchedule: VirtualUserScheduleTable
   audit: AuditTable
 }

package/src/kysely-virtual-user-run-store.test.ts

@@ -39,6 +39,28 @@ const FINDING = {
   step: 7,
 } as any
 
+const INTENT = {
+  id: 'intent_1',
+  sourceId: 'exportADoc',
+  title: 'Export a doc',
+  status: 'done',
+  steps: [0, 1],
+  suspensions: 0,
+} as any
+
+const step = (index: number, over: Record<string, unknown> = {}) =>
+  ({
+    index,
+    intentId: 'intent_1',
+    action: { kind: 'call', rpcName: 'listDocs', input: {} },
+    status: 200,
+    ok: true,
+    response: '{"docs":[]}',
+    tokensIn: 40,
+    tokensOut: 12,
+    ...over,
+  }) as any
+
 describe('KyselyVirtualUserRunStore', () => {
   let db: Kysely<KyselyPikkuDB>
   let store: KyselyVirtualUserRunStore
@@ -114,6 +136,8 @@ describe('KyselyVirtualUserRunStore', () => {
       tally: TALLY,
       memory: { docId: 'doc_1', orgId: 'org_9' },
       stoppedBy: 'budget-steps',
+      intents: [INTENT],
+      steps: [step(0), step(1)],
     })
 
     const run = await store.get(runId)
@@ -182,14 +206,17 @@ describe('KyselyVirtualUserRunStore', () => {
     assert.equal(runs[0]?.persona, 'susan')
   })
 
-  test('completing an unknown run does not throw', async () => {
+  test('completing an unknown run does not throw, and leaves no steps behind', async () => {
     await store.init()
     await store.complete('nope', {
       findings: [],
       tally: TALLY,
       memory: {},
       stoppedBy: 'exhausted',
+      intents: [INTENT],
+      steps: [step(0)],
     })
+    assert.deepEqual(await store.steps('nope'), [])
   })
 
   // Not every project installs SerializePlugin, and a bare SQLite driver cannot
@@ -214,6 +241,8 @@ describe('KyselyVirtualUserRunStore', () => {
       tally: TALLY,
       memory: { docId: 'doc_1' },
       stoppedBy: 'exhausted',
+      intents: [INTENT],
+      steps: [step(0)],
     })
 
     const run = await bareStore.get(runId)
@@ -224,5 +253,117 @@ describe('KyselyVirtualUserRunStore', () => {
     assert.equal(run?.findings[0]?.detail, FINDING.detail)
     assert.ok(run?.createdAt instanceof Date)
     assert.ok(run?.finishedAt instanceof Date)
+    assert.deepEqual(await bareStore.steps(runId), [step(0)])
+  })
+
+  test('the transcript comes back in the order it happened', async () => {
+    const runId = await store.start({
+      persona: 'susan',
+      disposition: 'realistic',
+      seed: 1,
+    })
+
+    await store.complete(runId, {
+      findings: [],
+      tally: TALLY,
+      memory: {},
+      stoppedBy: 'exhausted',
+      intents: [INTENT],
+      steps: [step(2), step(0), step(1)],
+    })
+
+    const steps = await store.steps(runId)
+    assert.deepEqual(
+      steps.map((one) => one.index),
+      [0, 1, 2]
+    )
+    assert.deepEqual(steps[0], step(0))
+  })
+
+  // The turn the model got wrong is the one worth reading, and it carries none
+  // of the fields a successful call does.
+  test('a turn with no response and no status survives the round trip', async () => {
+    const runId = await store.start({
+      persona: 'susan',
+      disposition: 'careless',
+      seed: 1,
+    })
+
+    const invalid = step(0, {
+      intentId: undefined,
+      action: { kind: 'invalid', detail: 'no such rpc: listDoc' },
+      status: undefined,
+      ok: undefined,
+      response: undefined,
+      findingKinds: ['invalid-action'],
+    })
+
+    await store.complete(runId, {
+      findings: [],
+      tally: TALLY,
+      memory: {},
+      stoppedBy: 'exhausted',
+      intents: [],
+      steps: [invalid],
+    })
+
+    const [stored] = await store.steps(runId)
+    assert.deepEqual(stored, {
+      index: 0,
+      action: { kind: 'invalid', detail: 'no such rpc: listDoc' },
+      findingKinds: ['invalid-action'],
+      tokensIn: 40,
+      tokensOut: 12,
+    })
+  })
+
+  // Ten columns times a 500-step budget is five thousand bound variables, and a
+  // bare sqlite driver takes 999 — so the long run is exactly the one that must
+  // not be lost.
+  test('a run longer than one insert can hold is stored whole', async () => {
+    const runId = await store.start({
+      persona: 'susan',
+      disposition: 'adversarial',
+      seed: 1,
+    })
+
+    const steps = Array.from({ length: 240 }, (_, index) => step(index))
+    await store.complete(runId, {
+      findings: [],
+      tally: TALLY,
+      memory: {},
+      stoppedBy: 'budget-steps',
+      intents: [],
+      steps,
+    })
+
+    assert.equal((await store.steps(runId, { limit: 500 })).length, 240)
+    assert.deepEqual(
+      (await store.steps(runId, { limit: 2, offset: 100 })).map(
+        (one) => one.index
+      ),
+      [100, 101]
+    )
+  })
+
+  test('intents ride on the run, not on the transcript', async () => {
+    const runId = await store.start({
+      persona: 'susan',
+      disposition: 'realistic',
+      seed: 1,
+    })
+
+    await store.complete(runId, {
+      findings: [],
+      tally: TALLY,
+      memory: {},
+      stoppedBy: 'exhausted',
+      intents: [INTENT],
+      steps: [],
+    })
+
+    const [listed] = await store.list({ persona: 'susan' })
+    assert.deepEqual(listed?.intents, [INTENT])
+    assert.deepEqual(await store.steps(runId), [])
   })
 })

package/src/kysely-virtual-user-run-store.ts

@@ -1,13 +1,20 @@
 import type { Kysely, Selectable } from 'kysely'
 import { randomUUID } from 'node:crypto'
-import type { VirtualUserDisposition } from '@pikku/core/virtual-user'
+import type {
+  StepRecord,
+  VirtualUserDisposition,
+} from '@pikku/core/virtual-user'
 import type {
   VirtualUserRunOutcome,
   VirtualUserRunRecord,
   VirtualUserRunStart,
   VirtualUserRunStore,
 } from '@pikku/core/virtual-user'
-import type { KyselyPikkuDB, VirtualUserRunTable } from './kysely-tables.js'
+import type {
+  KyselyPikkuDB,
+  VirtualUserRunStepTable,
+  VirtualUserRunTable,
+} from './kysely-tables.js'
 import { parseJson } from './kysely-json.js'
 import { requirePikkuSchema } from './schema/index.js'
 import { virtualUserSchema } from './schema/virtual-user.schema.js'
@@ -31,6 +38,16 @@ import { virtualUserSchema } from './schema/virtual-user.schema.js'
  * ids. This is a privileged store; the scaffolded reads are scope-gated, and
  * exposing these rows more widely publishes your own exploits.
  */
+/**
+ * How many step rows go in one insert.
+ *
+ * Not a performance dial. A bare SQLite driver binds at most 999 variables per
+ * statement, and ten columns times a 500-step budget is five thousand — so an
+ * un-chunked insert does not fail on a long run, it fails on the *interesting*
+ * long run, which is the one nobody wants to lose.
+ */
+const STEP_INSERT_CHUNK = 50
+
 export class KyselyVirtualUserRunStore implements VirtualUserRunStore {
   private initialized = false
 
@@ -71,7 +88,7 @@ export class KyselyVirtualUserRunStore implements VirtualUserRunStore {
 
   async complete(runId: string, outcome: VirtualUserRunOutcome): Promise<void> {
     await this.init()
-    await this.db
+    const updated = await this.db
       .updateTable('virtualUserRun')
       .set({
         status: 'completed',
@@ -80,11 +97,16 @@ export class KyselyVirtualUserRunStore implements VirtualUserRunStore {
         // Overwritten rather than merged: the engine's memory already carries
         // what it was given, plus what it learned on the way.
         memory: JSON.stringify(outcome.memory),
+        intents: JSON.stringify(outcome.intents),
         stoppedBy: outcome.stoppedBy,
         finishedAt: new Date().toISOString(),
       })
       .where('runId', '=', runId)
-      .execute()
+      .executeTakeFirst()
+    // Nothing owns steps written against a run that is not there, and nothing
+    // would ever read or reap them — there is no foreign key to refuse them.
+    if (updated.numUpdatedRows === 0n) return
+    await this.writeSteps(runId, outcome.steps)
   }
 
   async fail(runId: string, error: string): Promise<void> {
@@ -128,6 +150,69 @@ export class KyselyVirtualUserRunStore implements VirtualUserRunStore {
     return rows.map((row) => this.toRecord(row))
   }
 
+  async steps(
+    runId: string,
+    options?: { limit?: number; offset?: number }
+  ): Promise<StepRecord[]> {
+    await this.init()
+    const rows = await this.db
+      .selectFrom('virtualUserRunStep')
+      .selectAll()
+      .where('runId', '=', runId)
+      .orderBy('stepIndex', 'asc')
+      .limit(options?.limit ?? 1000)
+      .offset(options?.offset ?? 0)
+      .execute()
+    return rows.map((row) => this.toStep(row))
+  }
+
+  private async writeSteps(
+    runId: string,
+    steps: readonly StepRecord[]
+  ): Promise<void> {
+    for (let at = 0; at < steps.length; at += STEP_INSERT_CHUNK) {
+      const chunk = steps.slice(at, at + STEP_INSERT_CHUNK)
+      await this.db
+        .insertInto('virtualUserRunStep')
+        .values(
+          chunk.map((step) => ({
+            runId,
+            stepIndex: step.index,
+            intentId: step.intentId ?? null,
+            action: JSON.stringify(step.action),
+            status: step.status ?? null,
+            ok: step.ok === undefined ? null : step.ok ? 1 : 0,
+            response:
+              step.response === undefined
+                ? null
+                : JSON.stringify(step.response),
+            findingKinds: step.findingKinds
+              ? JSON.stringify(step.findingKinds)
+              : null,
+            tokensIn: step.tokensIn,
+            tokensOut: step.tokensOut,
+          }))
+        )
+        .execute()
+    }
+  }
+
+  private toStep(row: Selectable<VirtualUserRunStepTable>): StepRecord {
+    return {
+      index: Number(row.stepIndex),
+      ...(row.intentId ? { intentId: row.intentId } : {}),
+      action: parseJson(row.action),
+      ...(row.status === null ? {} : { status: Number(row.status) }),
+      ...(row.ok === null ? {} : { ok: Boolean(row.ok) }),
+      ...(row.response === null ? {} : { response: parseJson(row.response) }),
+      ...(row.findingKinds
+        ? { findingKinds: parseJson(row.findingKinds) }
+        : {}),
+      tokensIn: Number(row.tokensIn),
+      tokensOut: Number(row.tokensOut),
+    }
+  }
+
   private toRecord(row: Selectable<VirtualUserRunTable>): VirtualUserRunRecord {
     return {
       runId: row.runId,
@@ -138,6 +223,7 @@ export class KyselyVirtualUserRunStore implements VirtualUserRunStore {
       goals: parseJson(row.goals) ?? [],
       memory: parseJson(row.memory) ?? {},
       findings: parseJson(row.findings) ?? [],
+      intents: parseJson(row.intents) ?? [],
       tally: row.tally ? (parseJson(row.tally) ?? null) : null,
       stoppedBy: row.stoppedBy,
       error: row.error,