@boxxy/pepys 0.0.1 → 0.1.3

package/src/queries/find-item-entity-id.ts

@@ -0,0 +1,16 @@
+import { buildEntityID, parseEntityID } from '@pepys/kit';
+import type { EntityID } from '@pepys/kit';
+
+// The item of a query's input as an entity ID. undefined: the text names no boxxy item, so the
+// query answers as for an item with no row.
+export function findItemEntityID(text: string): EntityID | undefined {
+  const parsed = parseEntityID(text);
+
+  if (!parsed.ok || parsed.value.source !== 'boxxy' || parsed.value.kind !== 'item') {
+    return undefined;
+  }
+
+  const built = buildEntityID(parsed.value);
+
+  return built.ok ? built.value : undefined;
+}

package/src/queries/types.ts

@@ -0,0 +1,168 @@
+import type {
+  ColumnKind,
+  ConsumerDeclaration,
+  EntityID,
+  QueryCall,
+  TableDeclaration,
+} from '@pepys/kit';
+import type { boxxyWorkConsumer } from '../work/boxxy-work-consumer.ts';
+
+// The rows the statements of the two queries read. Each holds the columns of its table that the
+// builders need, under the column names.
+
+export interface AttemptReadRow {
+  id: string;
+  phase: string;
+  number: number;
+  round: number;
+  state: string;
+  outcome: string | null;
+  failure: string | null;
+  claimed_at: string;
+  ended_at: string | null;
+
+  // a JSON column: the list of sleep stretches, as text or as the parsed list
+  sleeps: unknown;
+}
+
+// A row of `boxxy_attempt_session` for every session that an attempt of the item used, with the
+// rows of the other attempts, of any item, that used the same session.
+export interface BindingReadRow {
+  attempt_id: string;
+  session_id: string;
+  bound_at: string;
+  caller_issuer: string | null;
+  caller_subject: string | null;
+}
+
+// the proven sender of a telemetry row, and its session and time
+interface TelemetryReadRow {
+  session_id: string;
+  origin_issuer: string | null;
+  origin_subject: string | null;
+  product: string;
+  at: string;
+}
+
+export interface RequestReadRow extends TelemetryReadRow {
+  model: string;
+  cost_basis: string;
+  cost_usd: number | null;
+  input_tokens: number;
+  cache_read_tokens: number;
+  cache_creation_tokens: number;
+  output_tokens: number;
+}
+
+export interface TurnReadRow extends TelemetryReadRow {
+  model: string | null;
+  cost_usd: number;
+}
+
+// one pull request that an attempt of the item produced
+export interface PullRequestReadRow {
+  id: string;
+  merged_at: string | null;
+}
+
+// One pull request of the item with its delivery times, as the GitHub plug-in's delivery module
+// gives them. Two attempts that produced one pull request give two equal rows.
+export interface DeliveryReadRow {
+  pullRequest: string;
+  mergedAt: string | null;
+  release: { at: string } | null;
+  deploy: { at: string } | null;
+}
+
+interface Column<Kind extends ColumnKind> {
+  readonly kind: Kind;
+}
+
+interface NullableColumn<Kind extends ColumnKind> {
+  readonly kind: Kind;
+  readonly nullable: true;
+}
+
+// The columns that the queries read of each consumer of another plug-in. A consumer with more
+// columns fits. A table's columns take a type alias: an interface has no index signature.
+type TelemetryColumns = Readonly<{
+  session_id: Column<'entityID'>;
+  origin_issuer: NullableColumn<'text'>;
+  origin_subject: NullableColumn<'text'>;
+  product: Column<'text'>;
+  at: Column<'timestamp'>;
+}>;
+
+export type TelemetryModelRequestConsumer = ConsumerDeclaration<
+  'telemetry_model_request',
+  {
+    readonly telemetry_model_request: TableDeclaration<
+      TelemetryColumns & {
+        readonly model: Column<'text'>;
+        readonly cost_basis: Column<'text'>;
+        readonly cost_usd: NullableColumn<'real'>;
+        readonly input_tokens: Column<'integer'>;
+        readonly cache_read_tokens: Column<'integer'>;
+        readonly cache_creation_tokens: Column<'integer'>;
+        readonly output_tokens: Column<'integer'>;
+      }
+    >;
+  }
+>;
+
+export type TelemetryTurnCostConsumer = ConsumerDeclaration<
+  'telemetry_turn_cost',
+  {
+    readonly telemetry_turn_cost: TableDeclaration<
+      TelemetryColumns & {
+        readonly model: NullableColumn<'text'>;
+        readonly cost_usd: Column<'real'>;
+      }
+    >;
+  }
+>;
+
+export type GithubPullRequestConsumer = ConsumerDeclaration<
+  'github_pull_request',
+  {
+    readonly github_pull_request: TableDeclaration<{
+      readonly id: Column<'entityID'>;
+      readonly merged_at: NullableColumn<'timestamp'>;
+    }>;
+  }
+>;
+
+// A consumer whose tables the stage-time query does not type: the delivery read types its own,
+// so the query's statement on the boxxy tables keeps one table type. No table name holds a colon,
+// so the key pattern matches no real table and any consumer fits.
+export type OpaqueConsumer = ConsumerDeclaration<
+  string,
+  Readonly<Record<`opaque:${string}`, TableDeclaration>>
+>;
+
+export type StageTimeReads = readonly [typeof boxxyWorkConsumer, ...OpaqueConsumer[]];
+
+export interface DeliveryInput {
+  readonly item: EntityID;
+
+  // the environment whose deploy counts
+  readonly environment: string;
+}
+
+// The GitHub plug-in's delivery times of each pull request that an attempt of the item produced,
+// with the consumers that hold them.
+export interface DeliveryReads {
+  readonly consumers: readonly OpaqueConsumer[];
+  readonly readDeliveries: (
+    db: QueryCall<unknown, StageTimeReads>['db'],
+    input: DeliveryInput,
+  ) => Promise<readonly DeliveryReadRow[]>;
+}
+
+// The consumers of the GitHub and telemetry plug-ins that the two queries read.
+export interface ForeignReads {
+  readonly telemetryModelRequest: TelemetryModelRequestConsumer;
+  readonly telemetryTurnCost: TelemetryTurnCostConsumer;
+  readonly githubPullRequest: GithubPullRequestConsumer;
+  readonly delivery: DeliveryReads;
+}

package/src/sink/is-outgoing-fact.ts

@@ -0,0 +1,19 @@
+import type { FactBatchBody, OutgoingFact } from '@pepys/kit';
+import type { boxxySource } from '../boxxy-source.ts';
+import { workFamily } from '../boxxy-source.ts';
+
+// The check that ingest makes of each fact of a batch: a type of the family, an entity of the
+// family's kind, and a payload that the type's schema accepts.
+export function isOutgoingFact(
+  fact: FactBatchBody['facts'][number],
+): fact is OutgoingFact<typeof boxxySource> {
+  return (
+    isFamilyType(fact.type) &&
+    fact.entity.startsWith(`${workFamily.entityKind}:`) &&
+    workFamily.types[fact.type].safeParse(fact.payload).success
+  );
+}
+
+function isFamilyType(type: string): type is keyof typeof workFamily.types {
+  return Object.hasOwn(workFamily.types, type);
+}

package/src/sink/pick-sink-answer.ts

@@ -0,0 +1,29 @@
+import type { SinkAnswer } from '@boxxy/model';
+import type { Result, SendFailure } from '@pepys/kit';
+
+type SendResult = Result<{ stored: number }, SendFailure>;
+
+// A refusal with 400, 413 or 422 is the batch's content, which a split can find. Ingest answers
+// 500 when a source throws, it has no blob store, or its log refuses an append: no split fixes
+// those, so a 500 blocks the outbox like any other refusal of the sink.
+export function pickSinkAnswer(sent: SendResult | 'timeout', timeoutMs: number): SinkAnswer {
+  if (sent === 'timeout') {
+    return { kind: 'retry', reason: `no answer in ${timeoutMs} ms` };
+  }
+
+  if (sent.ok) {
+    return { kind: 'stored' };
+  }
+
+  const failure = sent.failure;
+
+  if (failure.kind === 'retry') {
+    return { kind: 'retry', reason: failure.reason };
+  }
+
+  const status = failure.status;
+
+  return status === 400 || status === 413 || status === 422
+    ? { kind: 'refused-batch', status, reason: failure.reason }
+    : { kind: 'refused-sink', status, reason: failure.reason };
+}

package/src/sink/to-wire-fact.ts

@@ -0,0 +1,31 @@
+import type { OutboxFact } from '@boxxy/model';
+import type { FactBatchBody } from '@pepys/kit';
+
+type WireFact = FactBatchBody['facts'][number];
+
+// One outbox row as one fact of a batch. The event is the row's fact ID on every send, so a
+// resend is a copy that pepys drops; ingest makes the fact ID `event:boxxy:<event>`. The store
+// version goes into the payload here and nowhere else.
+export function toWireFact(fact: OutboxFact): WireFact {
+  return {
+    type: fact.type,
+    event: fact.factId,
+    entity: fact.entity,
+    actor: fact.actor,
+    at: new Date(fact.at).toISOString(),
+    payload: { ...toPayloadFields(fact.payload), storeVersion: fact.seq },
+  };
+}
+
+type Payload = OutboxFact['payload'];
+
+// Every catalog payload is an object; `Array.isArray` does not narrow a readonly list.
+function toPayloadFields(payload: Payload): Readonly<Record<string, unknown>> {
+  return isPayloadObject(payload) ? payload : {};
+}
+
+function isPayloadObject(
+  payload: Payload,
+): payload is Extract<Payload, Readonly<Record<string, unknown>>> {
+  return typeof payload === 'object' && payload !== null && !Array.isArray(payload);
+}

package/src/types.ts

@@ -0,0 +1,14 @@
+import type { PluginClient } from '@pepys/kit';
+import type { buildBoxxyPlugin } from './build-boxxy-plugin.ts';
+
+// The typed client of the plug-in's two queries on the pepys API's RPC route.
+export type BoxxyQueryClient = PluginClient<ReturnType<typeof buildBoxxyPlugin>>;
+
+// The same client as the reader takes it. Its types hold only when pepys runs this package's
+// version, so the reader parses each answer. Any `BoxxyQueryClient` fits, and so does the pepys
+// API's deployment client of a deployment that holds the plug-in.
+export type UncheckedQueryClient = {
+  readonly [Query in keyof BoxxyQueryClient]: (
+    input: Parameters<BoxxyQueryClient[Query]>[0],
+  ) => Promise<unknown>;
+};

package/src/wire-fact-schemas.ts

@@ -0,0 +1,41 @@
+import { workFactSchemas } from '@boxxy/model';
+import type { FactType } from '@boxxy/model';
+import { z } from 'zod';
+
+// The sink adds the store version to every payload, and no model schema holds it, so the wire
+// schema of a type is the model's schema plus that one field. These are the `types` of the fact
+// family.
+const storeVersion = { storeVersion: z.number().int().positive() };
+
+export const wireFactSchemas = {
+  'boxxy.item.filed': workFactSchemas['boxxy.item.filed'].extend(storeVersion),
+  'boxxy.item.noted': workFactSchemas['boxxy.item.noted'].extend(storeVersion),
+  'boxxy.item.edited': workFactSchemas['boxxy.item.edited'].extend(storeVersion),
+  'boxxy.item.closed': workFactSchemas['boxxy.item.closed'].extend(storeVersion),
+  'boxxy.relation.added': workFactSchemas['boxxy.relation.added'].extend(storeVersion),
+  'boxxy.relation.removed': workFactSchemas['boxxy.relation.removed'].extend(storeVersion),
+  'boxxy.lever.recorded': workFactSchemas['boxxy.lever.recorded'].extend(storeVersion),
+  'boxxy.attempt.claimed': workFactSchemas['boxxy.attempt.claimed'].extend(storeVersion),
+  'boxxy.decision.recorded': workFactSchemas['boxxy.decision.recorded'].extend(storeVersion),
+  'boxxy.attempt.started': workFactSchemas['boxxy.attempt.started'].extend(storeVersion),
+  'boxxy.attempt.spawn_unanswered':
+    workFactSchemas['boxxy.attempt.spawn_unanswered'].extend(storeVersion),
+  'boxxy.attempt.reconciled': workFactSchemas['boxxy.attempt.reconciled'].extend(storeVersion),
+  'boxxy.attempt.session_bound':
+    workFactSchemas['boxxy.attempt.session_bound'].extend(storeVersion),
+  'boxxy.attempt.slept': workFactSchemas['boxxy.attempt.slept'].extend(storeVersion),
+  'boxxy.attempt.wake_requested':
+    workFactSchemas['boxxy.attempt.wake_requested'].extend(storeVersion),
+  'boxxy.attempt.woken': workFactSchemas['boxxy.attempt.woken'].extend(storeVersion),
+  'boxxy.attempt.resumed': workFactSchemas['boxxy.attempt.resumed'].extend(storeVersion),
+  'boxxy.attempt.closed_out': workFactSchemas['boxxy.attempt.closed_out'].extend(storeVersion),
+  'boxxy.attempt.timed_out': workFactSchemas['boxxy.attempt.timed_out'].extend(storeVersion),
+  'boxxy.attempt.turn_ended': workFactSchemas['boxxy.attempt.turn_ended'].extend(storeVersion),
+  'boxxy.attempt.settled': workFactSchemas['boxxy.attempt.settled'].extend(storeVersion),
+  'boxxy.proposal.recorded': workFactSchemas['boxxy.proposal.recorded'].extend(storeVersion),
+  'boxxy.evidence.recorded': workFactSchemas['boxxy.evidence.recorded'].extend(storeVersion),
+  'boxxy.pull_request.linked': workFactSchemas['boxxy.pull_request.linked'].extend(storeVersion),
+  'boxxy.commit.linked': workFactSchemas['boxxy.commit.linked'].extend(storeVersion),
+  'boxxy.pull_request.merged': workFactSchemas['boxxy.pull_request.merged'].extend(storeVersion),
+  'boxxy.pull_request.closed': workFactSchemas['boxxy.pull_request.closed'].extend(storeVersion),
+} satisfies Readonly<Record<FactType, z.ZodObject>>;

package/src/work/boxxy-work-consumer.ts

@@ -0,0 +1,95 @@
+import { defineConsumer, defineTable } from '@pepys/kit';
+import { workFamily } from '../boxxy-source.ts';
+import { attemptOf, produced, ranIn } from './constants.ts';
+
+// One scope per item: a fact on the stream marks its item dirty, and the entry file gets every
+// fact of the item and returns every row and link the item owns.
+export const boxxyWorkConsumer = defineConsumer({
+  name: 'boxxy_work',
+  entry: new URL('../consumers/apply-scope.ts', import.meta.url),
+  input: { mode: 'scope', scopeKind: 'boxxy:item', streams: [workFamily.stream] },
+  tables: {
+    boxxy_item: defineTable({
+      columns: {
+        id: { kind: 'entityID' },
+        kind: { kind: 'text' },
+        repo: { kind: 'text' },
+        title: { kind: 'text' },
+        design_mode: { kind: 'text', nullable: true },
+        filed_at: { kind: 'timestamp' },
+        closed_at: { kind: 'timestamp', nullable: true },
+        close_reason: { kind: 'text', nullable: true },
+        updated_at: { kind: 'timestamp' },
+      },
+      primaryKey: ['id'],
+      indexes: [],
+    }),
+    boxxy_attempt: defineTable({
+      columns: {
+        id: { kind: 'entityID' },
+        item_id: { kind: 'entityID' },
+        phase: { kind: 'text' },
+        number: { kind: 'integer' },
+        round: { kind: 'integer' },
+        mode: { kind: 'text' },
+        pool: { kind: 'text' },
+        recipe_hash: { kind: 'text' },
+        rules_version: { kind: 'text' },
+        atc_session: { kind: 'text', nullable: true },
+        state: { kind: 'text' },
+        outcome: { kind: 'text', nullable: true },
+        failure: { kind: 'text', nullable: true },
+        claimed_at: { kind: 'timestamp' },
+        started_at: { kind: 'timestamp', nullable: true },
+        closed_out_at: { kind: 'timestamp', nullable: true },
+        ended_at: { kind: 'timestamp', nullable: true },
+
+        // the stretches of sleep, `{ from, to }` with `to` null while asleep: the entry file does
+        // no date arithmetic, so the queries add them up
+        sleeps: { kind: 'json' },
+        updated_at: { kind: 'timestamp' },
+      },
+      primaryKey: ['id'],
+      indexes: [{ columns: ['item_id'] }],
+    }),
+
+    // The join table of the cost query: one row per attempt and telemetry session. `key` `atc`
+    // holds the atc session, and `agent` the agent's own session, which matches only a telemetry
+    // row that carried no atc session ID.
+    boxxy_attempt_session: defineTable({
+      columns: {
+        attempt_id: { kind: 'entityID' },
+        session_id: { kind: 'entityID' },
+        key: { kind: 'text' },
+        product: { kind: 'text', nullable: true },
+        caller_issuer: { kind: 'text', nullable: true },
+        caller_subject: { kind: 'text', nullable: true },
+        bound_at: { kind: 'timestamp' },
+      },
+      primaryKey: ['attempt_id', 'session_id'],
+      indexes: [{ columns: ['session_id'] }],
+    }),
+  },
+  entityViews: [
+    {
+      table: 'boxxy_item',
+      kind: 'boxxy:item',
+      id: 'id',
+      title: 'title',
+      url: null,
+      actor: null,
+      updatedAt: 'updated_at',
+    },
+    {
+      table: 'boxxy_attempt',
+      kind: 'boxxy:attempt',
+      id: 'id',
+      title: 'phase',
+      url: null,
+      actor: null,
+      updatedAt: 'updated_at',
+    },
+  ],
+  reads: [],
+  links: [ranIn, produced, attemptOf],
+});

package/src/work/constants.ts

@@ -0,0 +1,25 @@
+import { defineLinkType } from '@pepys/kit';
+
+// `boxxy:attempt:<attempt ID>` to `telemetry:session:<atc or agent session ID>`
+export const ranIn = defineLinkType({
+  name: 'ran_in',
+  from: ['boxxy:attempt'],
+  to: ['telemetry:session'],
+  hub: false,
+});
+
+// `boxxy:attempt:<attempt ID>` to `github:pull_request:<GitHub's own ID of the pull request>`
+export const produced = defineLinkType({
+  name: 'produced',
+  from: ['boxxy:attempt'],
+  to: ['github:pull_request'],
+  hub: false,
+});
+
+// `boxxy:attempt:<attempt ID>` to `boxxy:item:<item ID>`
+export const attemptOf = defineLinkType({
+  name: 'attempt_of',
+  from: ['boxxy:attempt'],
+  to: ['boxxy:item'],
+  hub: false,
+});