@arcforgelabs/flow 2026.10.32 → 2026.10.34

package/server/core/reconcile.js

@@ -0,0 +1,350 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { and, desc, eq, gte, inArray, lte, sql } from 'drizzle-orm';
+import { backgroundJobs, billingObligations, invoices, reviewItems } from '../db/schema.js';
+import { HISTORY_ACTOR } from './history.js';
+import { soleOrganisation } from './pipeline.js';
+import { deadHoldSince, deadSince, enqueueJob, runNextJob } from './queue.js';
+import { openReview, resolveReviewFor } from './review.js';
+
+// The daily reconcile (issue 129, docs/RECONCILE.md). Flow checks each record
+// as it moves; once a day it also checks the whole. For each of the last
+// `days` days it counts and totals the jobs in three places, from Flow's own
+// tables only (no call to any other system):
+//
+// 1. the work system's copy (each adapter's `billableJobs`);
+// 2. the billing obligation ledger;
+// 3. the Flow invoices the ledger made, and the accounting system's copy of
+//    the invoices linked to them (`findInvoices`).
+//
+// A day where they do not agree opens one review item (`reconcile.day`) with
+// the counts, the totals and up to 20 record ids for each difference. A
+// difference is an alert only: the reconcile never changes a record. The item
+// closes by itself when a later run agrees.
+//
+// A difference first seen is checked again an hour later, and opens an item
+// only when it is still there: a job the pipeline has not reached yet is not
+// a difference.
+
+export const RECONCILE_KIND = 'reconcile.daily';
+export const RECONCILE_REVIEW_KIND = 'reconcile.day';
+export const ACTOR = 'system:reconcile';
+export const RECONCILE_DAYS = 60;
+export const EVIDENCE_IDS = 20;
+const RUN_EVERY_MS = 24 * 60 * 60 * 1000;
+const RECHECK_MS = 60 * 60 * 1000;
+const TICK_MS = 10 * 60 * 1000;
+const LEASE_MS = 15 * 60 * 1000;
+const FIND_BATCH = 400;
+// Accounting states where the ledger holds a link to an invoice that exists.
+const LINKED = ['drafted', 'authorised', 'part_paid', 'paid'];
+
+const isoDay = (ms) => new Date(ms).toISOString().slice(0, 10);
+const withGst = (cents) => Math.round(cents * 1.1);
+
+// A stable id for one day's review item, so a repeat finding refreshes it.
+export function dayReviewId(organisationId, day) {
+  const hex = createHash('sha256').update(`${RECONCILE_REVIEW_KIND}:${organisationId}:${day}`).digest('hex');
+  return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-5${hex.slice(13, 16)}-8${hex.slice(17, 20)}-${hex.slice(20, 32)}`;
+}
+
+function emptyDay(day) {
+  return {
+    day,
+    work: { count: 0, subtotalCents: 0, unpriced: 0 },
+    ledger: { count: 0, subtotalCents: 0, totalCents: 0 },
+    flowInvoices: { count: 0, totalCents: 0 },
+    accounting: { linked: 0, found: 0, totalCents: 0 },
+    differences: {
+      missingInLedger: [], missingInWork: [], missingInAccounting: [], accountingTotalDiffers: [],
+    },
+  };
+}
+
+const differenceCount = (entry) => Object.values(entry.differences).reduce((n, list) => n + list.length, 0);
+
+// The evidence a review item keeps: the counts, the totals, and up to
+// EVIDENCE_IDS ids for each difference with the full count beside them.
+function evidenceFor(entry) {
+  const differences = {};
+  for (const [key, list] of Object.entries(entry.differences)) {
+    if (list.length) differences[key] = { count: list.length, ids: list.slice(0, EVIDENCE_IDS).map((d) => d.id) };
+  }
+  return { ...entry, differences };
+}
+
+const moneyAtRisk = (entry) => Object.values(entry.differences).flat()
+  .reduce((cents, d) => cents + Math.abs(d.cents ?? 0), 0);
+
+// One pass over Flow's own tables. Answers one entry per day, oldest first.
+export async function reconcileDays(db, organisationId, { integrations, from, to }) {
+  const byDay = new Map();
+  const dayOf = (day) => {
+    if (!byDay.has(day)) byDay.set(day, emptyDay(day));
+    return byDay.get(day);
+  };
+
+  // 1. The work systems' copies.
+  const workAdapters = integrations?.with('billableJobs') ?? [];
+  const work = new Map(); // `${system}:${ref}` -> job
+  for (const adapter of workAdapters) {
+    const { jobs } = await adapter.capabilities.billableJobs({ organisationId, from, to });
+    for (const job of jobs) {
+      if (!job.day || job.day < from || job.day > to) continue;
+      work.set(`${adapter.id}:${job.ref}`, { ...job, system: adapter.id });
+      const entry = dayOf(job.day);
+      entry.work.count += 1;
+      if (Number.isInteger(job.subtotalCents)) entry.work.subtotalCents += job.subtotalCents;
+      else entry.work.unpriced += 1;
+    }
+  }
+
+  // 2. The ledger. A job only a history import reported is left out: the
+  // import keeps its own reconciliation record (docs/SEAZONA.md).
+  const obligations = await db.select({
+    id: billingObligations.id,
+    sourceSystem: billingObligations.sourceSystem,
+    sourceJobRef: billingObligations.sourceJobRef,
+    jobDate: billingObligations.jobDate,
+    subtotalCents: billingObligations.subtotalCents,
+    totalCents: billingObligations.totalCents,
+    accountingState: billingObligations.accountingState,
+    accountingRef: billingObligations.accountingRef,
+    invoiceId: billingObligations.invoiceId,
+    invoiceStatus: invoices.status,
+    invoiceTotalCents: invoices.totalCents,
+  })
+    .from(billingObligations)
+    .leftJoin(invoices, and(eq(invoices.organisationId, billingObligations.organisationId), eq(invoices.id, billingObligations.invoiceId)))
+    .where(and(
+      eq(billingObligations.organisationId, organisationId),
+      gte(billingObligations.jobDate, from),
+      lte(billingObligations.jobDate, to),
+      sql`exists (select 1 from obligation_events e
+        where e.organisation_id = ${billingObligations.organisationId} and e.obligation_id = ${billingObligations.id}
+          and e.kind in ('observed', 'source_changed') and e.actor <> ${HISTORY_ACTOR})`,
+    ));
+
+  const reported = new Set(workAdapters.map((a) => a.id));
+  const inLedger = new Set();
+  for (const o of obligations) {
+    const entry = dayOf(o.jobDate);
+    entry.ledger.count += 1;
+    entry.ledger.subtotalCents += o.subtotalCents;
+    entry.ledger.totalCents += o.totalCents;
+    if (o.invoiceId) {
+      entry.flowInvoices.count += 1;
+      entry.flowInvoices.totalCents += o.invoiceTotalCents ?? 0;
+    }
+    // Only a work system that is on can say a job is gone from its copy.
+    if (reported.has(o.sourceSystem)) {
+      const key = `${o.sourceSystem}:${o.sourceJobRef}`;
+      inLedger.add(key);
+      if (!work.has(key)) entry.differences.missingInWork.push({ id: o.id, cents: o.totalCents });
+    }
+  }
+  for (const [key, job] of work) {
+    if (!inLedger.has(key)) dayOf(job.day).differences.missingInLedger.push({ id: job.ref, cents: job.subtotalCents ?? 0 });
+  }
+
+  // 3. The accounting system's copy of each linked invoice.
+  const [accounting] = integrations?.with('findInvoices') ?? [];
+  const linked = obligations.filter((o) => o.accountingRef && LINKED.includes(o.accountingState));
+  if (accounting) {
+    const copies = new Map();
+    for (let i = 0; i < linked.length; i += FIND_BATCH) {
+      const invoiceIds = linked.slice(i, i + FIND_BATCH).map((o) => o.accountingRef);
+      const { invoices: found } = await accounting.capabilities.findInvoices({ organisationId, invoiceIds });
+      for (const invoice of found) copies.set(String(invoice.id), invoice);
+    }
+    for (const o of linked) {
+      const entry = dayOf(o.jobDate);
+      entry.accounting.linked += 1;
+      const copy = copies.get(String(o.accountingRef));
+      if (!copy) {
+        entry.differences.missingInAccounting.push({ id: o.id, cents: o.totalCents });
+        continue;
+      }
+      entry.accounting.found += 1;
+      entry.accounting.totalCents += copy.totalCents ?? 0;
+      // An invoice Flow issued must carry Flow's total. One Flow only matched
+      // may carry the job's total with or without GST, as the matcher allows.
+      const expected = o.invoiceId && o.invoiceStatus === 'issued'
+        ? [o.invoiceTotalCents]
+        : [o.totalCents, o.subtotalCents, withGst(o.subtotalCents)];
+      if (copy.totalCents !== null && !expected.includes(copy.totalCents)) {
+        entry.differences.accountingTotalDiffers.push({ id: o.id, cents: copy.totalCents - expected[0] });
+      }
+    }
+  }
+
+  return {
+    systems: { work: workAdapters.map((a) => a.id), accounting: accounting?.id ?? null },
+    days: [...byDay.values()].sort((a, b) => a.day.localeCompare(b.day)),
+  };
+}
+
+async function openDayItems(db, organisationId) {
+  const rows = await db.select({ flowId: reviewItems.flowId, evidence: reviewItems.evidence }).from(reviewItems).where(and(
+    eq(reviewItems.organisationId, organisationId),
+    eq(reviewItems.kind, RECONCILE_REVIEW_KIND),
+    eq(reviewItems.state, 'open'),
+  ));
+  return new Map(rows.map((r) => [r.evidence?.day, r.flowId]).filter(([day]) => day));
+}
+
+// One run: reconcile, open or refresh an item for each day that still
+// differs, and close the item of each day that now agrees. `unconfirmed` are
+// the days the last run found different for the first time. Answers the
+// summary the job keeps (GET /health shows it).
+export async function runReconcile(db, organisationId, {
+  integrations, now = Date.now(), days = RECONCILE_DAYS, unconfirmed = [],
+}) {
+  const to = isoDay(now);
+  const open = await openDayItems(db, organisationId);
+  // An open item older than the window is checked too, so it can close.
+  const oldest = [...open.keys()].reduce((min, day) => (day < min ? day : min), isoDay(now - (days - 1) * 86_400_000));
+  const { systems, days: entries } = await reconcileDays(db, organisationId, { integrations, from: oldest, to });
+  const differing = entries.filter((e) => differenceCount(e) > 0);
+  const recheck = new Set(unconfirmed);
+  const firstSeen = [];
+  let opened = 0;
+  for (const entry of differing) {
+    if (!open.has(entry.day) && !recheck.has(entry.day)) {
+      firstSeen.push(entry.day);
+      continue;
+    }
+    await openReview(db, organisationId, {
+      kind: RECONCILE_REVIEW_KIND,
+      entity: 'day',
+      flowId: dayReviewId(organisationId, entry.day),
+      systems: [...systems.work, ...(systems.accounting ? [systems.accounting] : [])],
+      evidence: evidenceFor(entry),
+      moneyAtRiskCents: moneyAtRisk(entry),
+    });
+    opened += 1;
+  }
+  const differs = new Set(differing.map((e) => e.day));
+  let closed = 0;
+  for (const [day, flowId] of open) {
+    if (differs.has(day)) continue;
+    await resolveReviewFor(db, organisationId, {
+      kind: RECONCILE_REVIEW_KIND, entity: 'day', flowId, resolution: 'a later run agreed', actor: ACTOR,
+    });
+    closed += 1;
+  }
+  return {
+    from: oldest,
+    to,
+    systems,
+    daysChecked: entries.length,
+    daysDifferent: differing.length,
+    openItems: (await openDayItems(db, organisationId)).size,
+    opened,
+    closed,
+    unconfirmed: firstSeen,
+  };
+}
+
+async function lastRun(db, organisationId) {
+  const [row] = await db.select().from(backgroundJobs).where(and(
+    eq(backgroundJobs.organisationId, organisationId),
+    eq(backgroundJobs.kind, RECONCILE_KIND),
+    inArray(backgroundJobs.state, ['succeeded', 'failed', 'dead']),
+  )).orderBy(desc(backgroundJobs.updatedAt)).limit(1);
+  return row ?? null;
+}
+
+// The last run and its result, for GET /health: counts only, no record ids.
+export async function reconcileHealth(db, { now = Date.now() } = {}) {
+  const organisationId = await soleOrganisation(db);
+  if (!organisationId) return null;
+  const row = await lastRun(db, organisationId);
+  if (!row) return { lastRunAt: null };
+  const result = row.state === 'succeeded' ? row.result ?? {} : {};
+  return {
+    lastRunAt: row.updatedAt.toISOString(),
+    state: row.state,
+    ...(row.state === 'succeeded' ? {
+      agrees: result.daysDifferent === 0,
+      daysChecked: result.daysChecked ?? null,
+      daysDifferent: result.daysDifferent ?? null,
+      openItems: result.openItems ?? null,
+      late: now - row.updatedAt.getTime() > RUN_EVERY_MS + 2 * TICK_MS + LEASE_MS,
+    } : { error: row.lastError }),
+  };
+}
+
+// Is a run due? Once a day, and an hour after a run that found a new difference.
+export function reconcileDue(last, nowMs) {
+  if (!last) return true;
+  const age = nowMs - last.updatedAt.getTime();
+  if (last.state !== 'succeeded') return age >= RECHECK_MS;
+  if ((last.result?.unconfirmed ?? []).length) return age >= RECHECK_MS;
+  return age >= RUN_EVERY_MS;
+}
+
+// The worker: every ten minutes it asks whether a run is due, and runs it on
+// the job queue, so two Flow processes never run it at once. It runs only
+// while an integration that feeds it is on.
+export function createReconcileWorker({
+  db, integrations, log = console, timers = { setTimeout, clearTimeout }, now = Date.now, everyMs = TICK_MS, firstDelayMs = 60_000,
+}) {
+  const workerId = `reconcile-${randomUUID()}`;
+  const startedAtMs = now();
+  let timer = null;
+  let running = null;
+  let stopped = true;
+
+  function schedule(delayMs) {
+    if (stopped) return;
+    timer = timers.setTimeout(tick, delayMs);
+    timer?.unref?.();
+  }
+
+  async function tick() {
+    timer = null;
+    if (stopped) return;
+    running = (async () => {
+      try {
+        if (!integrations?.with('billableJobs').length && !integrations?.with('findInvoices').length) return;
+        const organisationId = await soleOrganisation(db);
+        if (!organisationId) return;
+        const last = await lastRun(db, organisationId);
+        if (!reconcileDue(last, now())) return;
+        if (await deadSince(db, organisationId, { kind: RECONCILE_KIND, dedupeKey: 'all', since: deadHoldSince(startedAtMs, now()) })) return;
+        const unconfirmed = last?.state === 'succeeded' ? last.result?.unconfirmed ?? [] : [];
+        await enqueueJob(db, organisationId, { kind: RECONCILE_KIND, dedupeKey: 'all', maxAttempts: 3 });
+        const ran = await runNextJob(db, {
+          workerId,
+          leaseMs: LEASE_MS,
+          handlers: { [RECONCILE_KIND]: () => runReconcile(db, organisationId, { integrations, now: now(), unconfirmed }) },
+        });
+        if (!ran) return;
+        if (ran.outcome === 'dead') log.error?.('[reconcile] the daily reconcile gave up after its last attempt');
+        else if (ran.outcome === 'succeeded') {
+          const { daysChecked, daysDifferent, opened, closed } = ran.job.result;
+          (daysDifferent ? log.warn : log.info)?.(`[reconcile] ${daysChecked} days checked, ${daysDifferent} differ; ${opened} items open or refreshed, ${closed} closed`);
+        } else log.warn?.(`[reconcile] ${ran.outcome}: ${ran.job.lastError}`);
+      } catch (error) {
+        log.error?.(`[reconcile] ${error?.code ?? 'error'}: ${error?.message ?? error}`);
+      }
+    })();
+    await running;
+    running = null;
+    schedule(everyMs);
+  }
+
+  return {
+    start() {
+      stopped = false;
+      schedule(firstDelayMs);
+    },
+    async stop() {
+      stopped = true;
+      if (timer) timers.clearTimeout(timer);
+      timer = null;
+      await running;
+    },
+    tick,
+  };
+}

package/server/core/sending.js

@@ -1,9 +1,10 @@
 import { randomUUID } from 'node:crypto';
 import { and, asc, desc, eq, inArray, sql } from 'drizzle-orm';
-import { invoiceLines, invoices } from '../db/schema.js';
+import { billingObligations, invoiceLines, invoices } from '../db/schema.js';
 import { FlowError } from './errors.js';
 import { issueInvoice } from './invoices.js';
 import { soleOrganisation } from './pipeline.js';
+import { needsRead, readAccountingFirst, readBeforeAct, READS_PER_PASS } from './read-before-act.js';
 import { deadHoldSince, deadSince, enqueueJob, runNextJob } from './queue.js';
 
 // Drafts, Problem, Ready and Send all ready (docs/INVOICE_SENDING.md).
@@ -60,6 +61,15 @@ async function checkDrafts(db, organisationId, drafts, accounting) {
     const found = draftProblems(draft, linesOf.get(draft.id) ?? []);
     if (found.length) problems.set(draft.id, found);
   }
+  // A draft whose job the ledger sent to review (the job changed after the
+  // draft was made, or the work system deleted it) waits for a person.
+  const reviewed = await db.select({ invoiceId: billingObligations.invoiceId, reason: billingObligations.holdReason })
+    .from(billingObligations)
+    .where(and(eq(billingObligations.organisationId, organisationId), inArray(billingObligations.invoiceId, ids), eq(billingObligations.holdState, 'review')));
+  for (const { invoiceId, reason } of reviewed) {
+    const found = problem('job_review', `The job needs a look before this invoice goes: ${reason ?? 'it was sent to review'}.`);
+    problems.set(invoiceId, [...(problems.get(invoiceId) ?? []), found]);
+  }
   if (accounting) {
     const outside = await accounting.capabilities.checkDrafts({ organisationId, invoiceIds: ids });
     for (const [id, found] of outside ?? []) {
@@ -146,18 +156,46 @@ async function assertCanSend(accounting, organisationId) {
   }
 }
 
+// Read before act (issue #141): the jobs behind these drafts whose work
+// system report is old are read again first. Answers the drafts that may go
+// now; a draft whose job went to review, or could not be read, waits.
+async function readJobsFirst(db, organisationId, invoiceIds, { work, now, personId }) {
+  if (!invoiceIds.length || typeof work?.capabilities?.readWorkItem !== 'function') return { go: invoiceIds, waiting: [] };
+  const jobs = await db.select().from(billingObligations)
+    .where(and(eq(billingObligations.organisationId, organisationId), inArray(billingObligations.invoiceId, invoiceIds)));
+  const old = jobs.filter((job) => needsRead(job, { work, now }));
+  const outcomes = await readBeforeAct(db, organisationId, old.slice(0, READS_PER_PASS), { work, now, actor: `person:${personId}` });
+  const held = new Set(old.slice(READS_PER_PASS).map((job) => job.invoiceId));
+  for (const { outcome, obligation } of outcomes.values()) {
+    if (outcome === 'waiting' || outcome === 'review') held.add(obligation.invoiceId);
+  }
+  return { go: invoiceIds.filter((id) => !held.has(id)), waiting: invoiceIds.filter((id) => held.has(id)) };
+}
+
 // Send all ready: issues each ready draft, then queues it for the accounting
 // system. With `invoiceIds`, only those drafts, and each must be ready (the
-// Send button on one invoice). Answers { issued, queued, remaining }.
-export async function sendAllReady(db, organisationId, { accounting = null, personId, issueDate, invoiceIds = null }) {
+// Send button on one invoice). `work` is the work integration, for the read
+// before act. Answers { issued, queued, remaining, waiting }.
+export async function sendAllReady(db, organisationId, { accounting = null, work = null, personId, issueDate, invoiceIds = null, now = new Date() }) {
   await assertCanSend(accounting, organisationId);
-  let { ready } = await invoiceViews(db, organisationId, { accounting });
+  // Issue #45: the accounting copy reads its latest changes first, so the
+  // draft checks below see an invoice someone made there meanwhile.
+  let views = await invoiceViews(db, organisationId, { accounting });
+  const accountingRead = views.ready.length ? await readAccountingFirst(db, organisationId, { accounting, now }) : { outcome: 'fresh' };
+  if (accountingRead.changed) views = await invoiceViews(db, organisationId, { accounting });
+  let { ready } = views;
   if (invoiceIds) {
     const wanted = new Set(invoiceIds);
     ready = ready.filter((id) => wanted.has(id));
     if (ready.length !== wanted.size) throw new FlowError('not_ready', 'Only a ready invoice can be sent. Mark it ready first.');
   }
-  const batch = ready.slice(0, SEND_BATCH);
+  const { go, waiting } = accountingRead.outcome === 'waiting'
+    ? { go: [], waiting: ready.slice(0, SEND_BATCH) }
+    : await readJobsFirst(db, organisationId, ready.slice(0, SEND_BATCH), { work, now, personId });
+  if (invoiceIds && waiting.length) {
+    throw new FlowError('not_ready', 'Flow could not confirm the job with the work system or the accounting system just now, or the job changed. Nothing was sent. Look at the invoice, then try again.');
+  }
+  const batch = go;
   let issued = 0;
   for (const invoiceId of batch) {
     // Queued first: the worker sends only issued invoices, so a stop between
@@ -173,7 +211,7 @@ export async function sendAllReady(db, organisationId, { accounting = null, pers
     }
     issued += 1;
   }
-  return { issued, queued: accounting ? issued : 0, remaining: ready.length - batch.length };
+  return { issued, queued: accounting ? issued : 0, remaining: Math.max(0, ready.length - SEND_BATCH), waiting: waiting.length };
 }
 
 // Send again: issued invoices that failed go back on the queue. With no ids,

package/server/core/tracing.js

@@ -0,0 +1,64 @@
+import { routePattern } from '../ui/monitoring.js';
+
+// Performance traces for the installation's own tracing service
+// (docs/SENTRY.md, "Traces"). The first integration that is on and offers the
+// `traceSpan` and `traceRequest` capabilities gets them. With none on, each
+// function here runs the work as it is and adds nothing.
+//
+// `tracers()` answers the adapters to trace with; it is read at each call,
+// so an integration turned on or off later is followed.
+export function createTracing({ tracers = () => [] } = {}) {
+  const first = () => tracers()[0]?.capabilities ?? null;
+  return {
+    // Runs `work` inside a span named `name`, of kind `op`.
+    span(name, op, work, attributes = {}) {
+      const tracer = first();
+      return tracer ? tracer.traceSpan({ name, op, attributes }, work) : work();
+    },
+    // Express middleware: names each request's trace by its route pattern
+    // ("GET /api/invoicing/invoices/:id"), never by its URL, so the same
+    // click on another invoice is the same name.
+    requests() {
+      return (req, res, next) => {
+        const rename = first()?.traceRequest() ?? null;
+        if (rename) res.once('finish', () => rename(requestName(req)));
+        next();
+      };
+    },
+  };
+}
+
+// The route Express matched, else the path with its ids as ":id".
+export function requestName(req) {
+  const route = typeof req.route?.path === 'string' ? `${req.baseUrl ?? ''}${req.route.path}` : null;
+  return `${req.method} ${route ?? routePattern(req.originalUrl ?? req.url)}`;
+}
+
+// A span for each database query the pool runs. The span's name is the SQL
+// with its values left out: Flow passes values as parameters, and any quoted
+// text or number in the SQL itself becomes "?".
+export function traceQueries(pool, tracing) {
+  const wrap = (target) => {
+    const query = target.query.bind(target);
+    target.query = (...args) => {
+      // Callback calls are pg's own (the pool's query runs through a
+      // client's): the promise call around them already has its span.
+      if (typeof args.at(-1) === 'function') return query(...args);
+      return tracing.span(sqlName(args[0]), 'db', () => query(...args), { 'db.system': 'postgresql', 'db.system.name': 'postgresql' });
+    };
+  };
+  wrap(pool);
+  // A client a transaction holds (pool.connect()) queries directly.
+  pool.on('connect', wrap);
+  return pool;
+}
+
+export function sqlName(statement) {
+  const text = typeof statement === 'string' ? statement : String(statement?.text ?? '');
+  return text
+    .replace(/'(?:[^']|'')*'/g, '?')
+    .replace(/(?<![\w$])\d+(\.\d+)?\b/g, '?')
+    .replace(/\s+/g, ' ')
+    .trim()
+    .slice(0, 300) || 'query';
+}

package/server/db/migrations/0017_sync_basis.sql

@@ -0,0 +1 @@
+ALTER TABLE "billing_obligations" ADD COLUMN IF NOT EXISTS "draft_basis" jsonb;

package/server/db/migrations/0018_custom_fields.sql

@@ -0,0 +1,78 @@
+CREATE TABLE IF NOT EXISTS "job_field_events" (
+	"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+	"organisation_id" uuid NOT NULL,
+	"field_id" uuid NOT NULL,
+	"source_system" text NOT NULL,
+	"source_job_ref" text NOT NULL,
+	"from_value" jsonb,
+	"to_value" jsonb,
+	"actor" uuid,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "job_field_values" (
+	"organisation_id" uuid NOT NULL,
+	"field_id" uuid NOT NULL,
+	"source_system" text NOT NULL,
+	"source_job_ref" text NOT NULL,
+	"value" jsonb NOT NULL,
+	"updated_by" uuid,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "job_field_values_organisation_id_field_id_source_system_source_job_ref_pk" PRIMARY KEY("organisation_id","field_id","source_system","source_job_ref")
+);
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "job_fields" (
+	"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+	"organisation_id" uuid NOT NULL,
+	"name" text NOT NULL,
+	"type" text NOT NULL,
+	"options" jsonb DEFAULT '[]'::jsonb NOT NULL,
+	"position" integer NOT NULL,
+	"archived_at" timestamp with time zone,
+	"created_by" uuid,
+	"updated_by" uuid,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "job_fields_org_id" UNIQUE("organisation_id","id"),
+	CONSTRAINT "job_fields_type_valid" CHECK ("job_fields"."type" in ('dropdown', 'person', 'checkbox', 'date', 'number', 'text')),
+	CONSTRAINT "job_fields_name_valid" CHECK (length("job_fields"."name") between 1 and 60)
+);
+--> statement-breakpoint
+DO $flow_guard$ BEGIN
+	ALTER TABLE "job_field_events" ADD CONSTRAINT "job_field_events_organisation_id_organisations_id_fk" FOREIGN KEY ("organisation_id") REFERENCES "public"."organisations"("id") ON DELETE no action ON UPDATE no action;
+EXCEPTION WHEN duplicate_object OR duplicate_table OR invalid_table_definition THEN NULL;
+END $flow_guard$;--> statement-breakpoint
+DO $flow_guard$ BEGIN
+	ALTER TABLE "job_field_events" ADD CONSTRAINT "job_field_events_actor_people_id_fk" FOREIGN KEY ("actor") REFERENCES "public"."people"("id") ON DELETE set null ON UPDATE no action;
+EXCEPTION WHEN duplicate_object OR duplicate_table OR invalid_table_definition THEN NULL;
+END $flow_guard$;--> statement-breakpoint
+DO $flow_guard$ BEGIN
+	ALTER TABLE "job_field_events" ADD CONSTRAINT "job_field_events_field_same_org" FOREIGN KEY ("organisation_id","field_id") REFERENCES "public"."job_fields"("organisation_id","id") ON DELETE cascade ON UPDATE no action;
+EXCEPTION WHEN duplicate_object OR duplicate_table OR invalid_table_definition THEN NULL;
+END $flow_guard$;--> statement-breakpoint
+DO $flow_guard$ BEGIN
+	ALTER TABLE "job_field_values" ADD CONSTRAINT "job_field_values_organisation_id_organisations_id_fk" FOREIGN KEY ("organisation_id") REFERENCES "public"."organisations"("id") ON DELETE no action ON UPDATE no action;
+EXCEPTION WHEN duplicate_object OR duplicate_table OR invalid_table_definition THEN NULL;
+END $flow_guard$;--> statement-breakpoint
+DO $flow_guard$ BEGIN
+	ALTER TABLE "job_field_values" ADD CONSTRAINT "job_field_values_updated_by_people_id_fk" FOREIGN KEY ("updated_by") REFERENCES "public"."people"("id") ON DELETE set null ON UPDATE no action;
+EXCEPTION WHEN duplicate_object OR duplicate_table OR invalid_table_definition THEN NULL;
+END $flow_guard$;--> statement-breakpoint
+DO $flow_guard$ BEGIN
+	ALTER TABLE "job_field_values" ADD CONSTRAINT "job_field_values_field_same_org" FOREIGN KEY ("organisation_id","field_id") REFERENCES "public"."job_fields"("organisation_id","id") ON DELETE cascade ON UPDATE no action;
+EXCEPTION WHEN duplicate_object OR duplicate_table OR invalid_table_definition THEN NULL;
+END $flow_guard$;--> statement-breakpoint
+DO $flow_guard$ BEGIN
+	ALTER TABLE "job_fields" ADD CONSTRAINT "job_fields_organisation_id_organisations_id_fk" FOREIGN KEY ("organisation_id") REFERENCES "public"."organisations"("id") ON DELETE no action ON UPDATE no action;
+EXCEPTION WHEN duplicate_object OR duplicate_table OR invalid_table_definition THEN NULL;
+END $flow_guard$;--> statement-breakpoint
+DO $flow_guard$ BEGIN
+	ALTER TABLE "job_fields" ADD CONSTRAINT "job_fields_created_by_people_id_fk" FOREIGN KEY ("created_by") REFERENCES "public"."people"("id") ON DELETE set null ON UPDATE no action;
+EXCEPTION WHEN duplicate_object OR duplicate_table OR invalid_table_definition THEN NULL;
+END $flow_guard$;--> statement-breakpoint
+DO $flow_guard$ BEGIN
+	ALTER TABLE "job_fields" ADD CONSTRAINT "job_fields_updated_by_people_id_fk" FOREIGN KEY ("updated_by") REFERENCES "public"."people"("id") ON DELETE set null ON UPDATE no action;
+EXCEPTION WHEN duplicate_object OR duplicate_table OR invalid_table_definition THEN NULL;
+END $flow_guard$;--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "job_field_events_job" ON "job_field_events" USING btree ("organisation_id","source_system","source_job_ref","created_at");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "job_field_values_job" ON "job_field_values" USING btree ("organisation_id","source_system","source_job_ref");