@bongos/core 1.17.0

package/modules/economy/cost.js

@@ -0,0 +1,438 @@
+'use strict';
+
+// Cost ledger + dashboard rollups — the cost_log writer (logCost), the
+// server-side session-cost reconciler, the costSummary/costCalibration
+// aggregations, the per-builder month-to-date budget status, and the
+// autonomous-spend MTD figure. Carved out of the GDS core src/bongos/db.js into
+// modules/economy/ (BV1.R77 / ADR 0093 §1-2) byte-for-byte — only the require()
+// paths changed (the shared pool + token pricing reach through the published
+// doorway; the cost-classify + cost-summary shapers are intra-module requires).
+// The lifecycle/reconciler resolve these through the `reward` kernel PORT rather
+// than importing this module.
+
+const { pool } = require('../../src/module-api');
+const { componentsFromModelUsage } = require('../../src/module-api').llmPricing; // reconciler (task 1366)
+const { classifyCostStatus } = require('./cost-classify');
+const costSummaryShape = require('./cost-summary-shape');
+
+// Shared `bounds`/`days` CTE for the costSummary day-series queries — spans
+// min(first cost day, first credit day) → CURRENT_DATE, one row per day, so an
+// idle day still gap-fills (the LEFT JOIN + COALESCE in each caller). Both
+// callers share this verbatim and differ only in the trailing `agg` CTE.
+const DAY_SERIES_CTE = `bounds AS (
+         SELECT LEAST(
+                  (SELECT MIN(date_trunc('day', recorded_at)::date) FROM cost_log),
+                  (SELECT MIN(date_trunc('day', recorded_at)::date) FROM credit_log WHERE delta > 0)
+                ) AS first_day,
+                CURRENT_DATE AS last_day
+       ),
+       days AS (
+         SELECT generate_series(first_day, last_day, interval '1 day')::date AS day
+           FROM bounds
+          WHERE first_day IS NOT NULL
+       )`;
+
+// V3.R84 (#302), criterion C3: a builder's month-to-date spend vs their monthly
+// budget, classified into ok/warn/over by the pure cost.classifyCostStatus().
+// MTD = cost_log rows attributed to this builder (migration 046 builder_id)
+// since date_trunc('month', now()). The budget is the per-builder override
+// (builders.monthly_budget_usd); NULL means uncapped — there is no default cap
+// (ADR 0085), so classifyCostStatus returns level 'ok'/budget null. This single
+// function backs the /me payload, the CLI session-start ping, the hall banner,
+// and the ship + art-pipeline hard-cap refusal — so every surface agrees.
+async function getBuilderCostStatus(builderId) {
+  const { rows } = await pool.query(
+    `SELECT
+        COALESCE((
+          SELECT SUM(amount_usd) FROM cost_log
+           WHERE builder_id = $1
+             AND recorded_at >= date_trunc('month', now())
+        ), 0) AS spend_mtd_usd,
+        (SELECT monthly_budget_usd FROM builders WHERE id = $1) AS budget_usd`,
+    [builderId]
+  );
+  const row = rows[0] || {};
+  return classifyCostStatus(row.spend_mtd_usd, row.budget_usd);
+}
+
+// System 3 (task 1351): project-wide AUTONOMOUS api spend month-to-date — the
+// `cron:%`-sourced rows (System 1 / task 1344 tags cron-launched sessions). The
+// autonomy precheck's hard spend-brake compares this against a ceiling so a
+// runaway autonomous fleet self-throttles. Returns a plain number (USD).
+async function autonomousApiSpendMtd() {
+  const { rows } = await pool.query(
+    `SELECT COALESCE(SUM(amount_usd), 0) AS usd
+       FROM cost_log
+      WHERE category = 'api'
+        AND source LIKE 'cron:%'
+        AND recorded_at >= date_trunc('month', now())`
+  );
+  return Number(rows[0]?.usd) || 0;
+}
+
+// V3.R84 (#302): Archon sets (or clears) a builder's monthly budget override.
+// usd === null clears the override (the builder falls back to the project
+// default). Returns the updated row's id + budget, or null if no such builder.
+async function setBuilderMonthlyBudget(builderId, usd) {
+  const { rows } = await pool.query(
+    `UPDATE builders SET monthly_budget_usd = $2
+      WHERE id = $1
+      RETURNING id, monthly_budget_usd`,
+    [builderId, usd]
+  );
+  return rows[0] ?? null;
+}
+
+// -------------------------------------------------------------------------
+// Cost log
+// -------------------------------------------------------------------------
+
+async function logCost({
+  taskId = null,
+  amountUsd,
+  category,
+  description = null,
+  source = null,
+  sourceRef = null,
+  recordedAt = null,
+  builderId = null,
+  // System 1 / task 1344: optional per-tier USD split (migration 123). NULL for
+  // non-LLM rows (manual/infra/compute); for the api session/cron rows written
+  // by session-cost.js these four sum to amount_usd by construction.
+  freshInputUsd = null,
+  cacheReadUsd = null,
+  cacheWriteUsd = null,
+  outputUsd = null,
+  // task #385 / migration 181: which skill invocation incurred this cost.
+  skillName = null,
+}) {
+  // V3.R11 (#91): builder_id is the per-builder attribution column added by
+  // migration 046. Routes default it to the authed builder, so the only
+  // legitimately-null builder_id is for historical rows or system-injected
+  // costs (infra, domain, etc.) that don't belong to any one person.
+  const { rows } = await pool.query(
+    `INSERT INTO cost_log (task_id, amount_usd, category, description, source, source_ref, recorded_at, builder_id,
+                           fresh_input_usd, cache_read_usd, cache_write_usd, output_usd, skill_name)
+     VALUES ($1, $2, $3, $4, $5, $6, COALESCE($7, now()), $8, $9, $10, $11, $12, $13)
+     ON CONFLICT (source, source_ref)
+       WHERE source IS NOT NULL AND source_ref IS NOT NULL
+       DO NOTHING
+     RETURNING id, recorded_at`,
+    [taskId, amountUsd, category, description, source, sourceRef, recordedAt, builderId,
+     freshInputUsd, cacheReadUsd, cacheWriteUsd, outputUsd, skillName || null]
+  );
+  return rows[0] ?? null; // null when ON CONFLICT skipped the insert
+}
+
+// task 1366 (System 1) — server-side session-cost reconciler.
+//
+// WHY THIS EXISTS. The client cost hook (scripts/gds/session-cost.js, a
+// SessionEnd hook) silently fails to write a cost_log row for the MAJORITY of
+// sessions: a cloud box / cowork VM / overnight agent runs with no GDS token,
+// so the hook quiet-skips, and some sessions post a digest but the cost POST
+// fails. ~1018 session_records in a 30-day window had a priceable model_usage
+// but NO matching cost_log row. The cure is NOT to harden the client — it's to
+// re-price the model_usage we ALREADY stored on session_records, server-side.
+//
+// We re-use componentsFromModelUsage (the same locked PRICES table as the rest
+// of the ledger) and logCost with source='session:interactive' and source_ref =
+// the bare session_id — the SAME (source, source_ref) key the client would use
+// — so logCost's ON CONFLICT (source, source_ref) DO NOTHING makes this a true
+// no-op if the client ALSO posted. There is no double-count path.
+//
+//   - olderThanMinutes : skip sessions newer than this so a healthy client gets
+//                        first crack at posting (avoids racing it). 15m default.
+//   - limit            : per-sweep cap (runaway guard; there are normally few).
+//
+// The NOT EXISTS checks BOTH the interactive key (session_id) AND the subagent
+// key (session_id || ':sub') — session-cost.js splits a captured session into
+// those two rows (task 1344), so a session the client captured even partially
+// is skipped entirely (we only have the combined model_usage, so we can't split
+// it; better to defer to the client's split than to add a duplicate total).
+// Returns { scanned, reconciled, totalUsd, unknownModels } for logging.
+async function reconcileSessionCosts({ olderThanMinutes = 15, limit = 500 } = {}) {
+  const mins = Number.isFinite(Number(olderThanMinutes)) ? Number(olderThanMinutes) : 15;
+  const lim = Number.isFinite(Number(limit)) ? Number(limit) : 500;
+  const { rows } = await pool.query(
+    `SELECT sr.session_id, sr.builder_id, sr.model_usage
+       FROM session_records sr
+      WHERE sr.model_usage IS NOT NULL
+        AND sr.uploaded_at < now() - ($1 || ' minutes')::interval
+        AND NOT EXISTS (
+          SELECT 1 FROM cost_log c
+           WHERE c.source_ref = sr.session_id
+              OR c.source_ref = sr.session_id || ':sub'
+        )
+      ORDER BY sr.uploaded_at DESC
+      LIMIT $2`,
+    [String(mins), lim]
+  );
+
+  const round4 = (n) => Math.round((Number(n) || 0) * 10000) / 10000;
+  let reconciled = 0;
+  let totalUsd = 0;
+  const unknownModels = new Set();
+  for (const r of rows) {
+    const comp = componentsFromModelUsage(r.model_usage);
+    for (const m of comp.unknown_models) unknownModels.add(m);
+    const amount = round4(comp.total_usd);
+    if (!(amount > 0)) continue; // nothing priceable on this row — skip
+    const id8 = String(r.session_id).slice(0, 8);
+    const unkNote = comp.unknown_models.length
+      ? ` [unpriced model(s) priced as opus: ${comp.unknown_models.join(',')}]`
+      : '';
+    const inserted = await logCost({
+      amountUsd: amount,
+      category: 'api',
+      source: 'session:interactive',
+      // SAME key the client would use → ON CONFLICT DO NOTHING prevents any
+      // double-count if the client also posts later.
+      sourceRef: r.session_id,
+      builderId: r.builder_id,
+      description: `session ${id8} — $${amount.toFixed(4)} (reconciled from digest; client cost-hook did not post)${unkNote}`.slice(0, 1000),
+      freshInputUsd: round4(comp.fresh_input_usd),
+      cacheReadUsd: round4(comp.cache_read_usd),
+      cacheWriteUsd: round4(comp.cache_write_usd),
+      outputUsd: round4(comp.output_usd),
+    });
+    // inserted === null means ON CONFLICT skipped it (client raced in between the
+    // SELECT and the INSERT) — expected + idempotent, not counted.
+    if (inserted) {
+      reconciled++;
+      totalUsd += amount;
+    }
+  }
+  return {
+    scanned: rows.length,
+    reconciled,
+    totalUsd: round4(totalUsd),
+    unknownModels: Array.from(unknownModels),
+  };
+}
+
+async function costSummary(filters = {}) {
+  // 1 credit == 1 USD per pms-v1.md; we report cost and value in the same
+  // numeric scale so the dashboard can plot them on a shared y-axis.
+  //
+  // task 1466: optional { source, category, days } filters let the status-page
+  // treasury be SLICED for analysis. They scope the BREAKDOWN reads only — the
+  // filtered subtotal, the by-category / by-builder / by-source chips, and the
+  // by-tier split. The header totals (total spent + value wrought), the day
+  // chart, and the value-wrought series stay PROJECT-WIDE (unfiltered): the
+  // filters drill into the breakdown beneath the chart, they don't rescope the
+  // whole summary. `filter_options` returns the full (unfiltered) source +
+  // category lists so the client can build stable dropdowns.
+  const cond = [];
+  const params = [];
+  if (filters.source) { params.push(filters.source); cond.push(`source = $${params.length}`); }
+  if (filters.category) { params.push(filters.category); cond.push(`category = $${params.length}`); }
+  const days = Number(filters.days);
+  const daysActive = Number.isFinite(days) && days > 0;
+  if (daysActive) { params.push(days); cond.push(`recorded_at >= now() - ($${params.length} * interval '1 day')`); }
+  const whereCl = cond.length ? `WHERE ${cond.join(' AND ')}` : '';
+  // byBuilder aliases cost_log as `cl`, so its predicates need the alias prefix.
+  const whereClPrefixed = cond.length ? `WHERE ${cond.map((c) => `cl.${c}`).join(' AND ')}` : '';
+  const [
+    { rows: totals },
+    { rows: byCat },
+    { rows: history },
+    { rows: valueTotals },
+    { rows: valueHistory },
+    { rows: byBuilder },
+    { rows: bySourceRows },
+    { rows: byTierRows },
+    { rows: filteredTotalRows },
+    { rows: sourceOptRows },
+    { rows: categoryOptRows },
+  ] = await Promise.all([
+    pool.query(`SELECT COALESCE(SUM(amount_usd), 0) AS total_usd FROM cost_log`),
+    pool.query(
+      `SELECT category, COALESCE(SUM(amount_usd), 0) AS total_usd
+         FROM cost_log ${whereCl} GROUP BY category ORDER BY total_usd DESC`,
+      params
+    ),
+    // Day series for costs — generated from min(first_cost_day, first_credit_day)
+    // through today (server timezone) so the chart line carries forward on
+    // idle days and always reaches today. LEFT JOIN + COALESCE means days
+    // with no rows return amount_usd=0; the client's running-sum walk turns
+    // those zeros into a flat-line carry-forward. GDS #118.
+    pool.query(
+      `WITH ${DAY_SERIES_CTE},
+       agg AS (
+         SELECT date_trunc('day', recorded_at)::date AS day,
+                SUM(amount_usd) AS amount_usd
+           FROM cost_log
+          GROUP BY day
+       )
+       SELECT d.day, COALESCE(a.amount_usd, 0) AS amount_usd
+         FROM days d
+         LEFT JOIN agg a ON a.day = d.day
+        ORDER BY d.day`
+    ),
+    pool.query(
+      `SELECT COALESCE(SUM(delta), 0) AS total_credits
+         FROM credit_log WHERE delta > 0`
+    ),
+    // Same gap-fill treatment for the value-wrought series. GDS #118.
+    pool.query(
+      `WITH ${DAY_SERIES_CTE},
+       agg AS (
+         SELECT date_trunc('day', recorded_at)::date AS day,
+                SUM(delta) AS credits
+           FROM credit_log
+          WHERE delta > 0
+          GROUP BY day
+       )
+       SELECT d.day, COALESCE(a.credits, 0) AS credits
+         FROM days d
+         LEFT JOIN agg a ON a.day = d.day
+        ORDER BY d.day`
+    ),
+    // V3.R11 (#91): cost split per builder. LEFT JOIN so rows with NULL
+    // builder_id (historical pre-migration-046 rows, infra/domain spend, etc.)
+    // surface as 'unattributed' rather than disappearing. Only builders with
+    // non-zero spend are returned; the dashboard renders these as a small
+    // breakdown below the by-track chips.
+    pool.query(
+      `SELECT cl.builder_id,
+              b.github_login,
+              b.display_name,
+              COALESCE(SUM(cl.amount_usd), 0) AS total_usd
+         FROM cost_log cl
+         LEFT JOIN builders b ON b.id = cl.builder_id
+         ${whereClPrefixed}
+         GROUP BY cl.builder_id, b.github_login, b.display_name
+         HAVING COALESCE(SUM(cl.amount_usd), 0) > 0
+         ORDER BY total_usd DESC`,
+      params
+    ),
+    // System 1 / task 1345: by_source — every cost_log row grouped by its
+    // call-site source tag (session:interactive / session:subagent / cron:* /
+    // grader / box-lifecycle / manual / …). The dimension that turns the
+    // "interactive marathons dominate" claim into a number.
+    pool.query(
+      `SELECT source, COALESCE(SUM(amount_usd), 0) AS total_usd
+         FROM cost_log ${whereCl} GROUP BY source ORDER BY total_usd DESC`,
+      params
+    ),
+    // by_tier — SUM of the per-tier columns (migration 123). Only session/cron
+    // rows carry the split; legacy/manual rows are NULL and SUM to 0 there.
+    pool.query(
+      `SELECT COALESCE(SUM(fresh_input_usd), 0) AS fresh_input_usd,
+              COALESCE(SUM(cache_read_usd), 0)  AS cache_read_usd,
+              COALESCE(SUM(cache_write_usd), 0) AS cache_write_usd,
+              COALESCE(SUM(output_usd), 0)      AS output_usd
+         FROM cost_log ${whereCl}`,
+      params
+    ),
+    // task 1466: the filtered subtotal — SUM of spend under the active filter,
+    // so the client can show "showing $X for <filter>" next to the chips
+    // without touching the project-wide header total above.
+    pool.query(
+      `SELECT COALESCE(SUM(amount_usd), 0) AS total_usd FROM cost_log ${whereCl}`,
+      params
+    ),
+    // task 1466: unfiltered option lists for the filter dropdowns, so the
+    // controls stay stable regardless of the current slice.
+    pool.query(
+      `SELECT DISTINCT source FROM cost_log WHERE source IS NOT NULL ORDER BY source`
+    ),
+    pool.query(
+      `SELECT DISTINCT category FROM cost_log WHERE category IS NOT NULL ORDER BY category`
+    ),
+  ]);
+
+  // savings — the counterfactual ledger (cost_baseline, migration 124). Guarded:
+  // the table is the newest in the deploy; if costSummary somehow runs before the
+  // migration applies, the public endpoint must still return (status dashboard
+  // reads it), so a missing table degrades to empty savings rather than a 500.
+  let savingsRows = [];
+  try {
+    const r = await pool.query(
+      `SELECT system,
+              COALESCE(SUM(baseline_usd), 0) AS baseline_usd,
+              COALESCE(SUM(amount_usd), 0)   AS amount_usd,
+              COUNT(*)                        AS events
+         FROM cost_baseline
+        GROUP BY system
+        ORDER BY (COALESCE(SUM(baseline_usd),0) - COALESCE(SUM(amount_usd),0)) DESC`
+    );
+    savingsRows = r.rows;
+  } catch (e) {
+    if (e && e.code !== '42P01') throw e; // 42P01 = undefined_table (pre-migration); anything else is real
+    console.error('[costSummary] cost_baseline not present yet — savings empty until migration 124 deploys');
+  }
+
+  // cost_calibration — cost-per-task / cost-per-credit medians (migration 143,
+  // spend-audit F6). Same guard posture as savings: the function is the newest in
+  // the deploy, so if costSummary runs before migration 143 applies the public
+  // endpoint must still return — a missing function degrades to empty calibration
+  // rather than a 500. 42883 = undefined_function (pre-migration).
+  let calibrationRows = [];
+  try {
+    const r = await pool.query('SELECT * FROM cost_calibration_summary()');
+    calibrationRows = r.rows;
+  } catch (e) {
+    if (e && e.code !== '42883') throw e;
+    console.error('[costSummary] cost_calibration_summary() not present yet — calibration empty until migration 143 deploys');
+  }
+
+  return {
+    total_usd: Number(totals[0].total_usd),
+    total_value_credits: Number(valueTotals[0].total_credits),
+    by_category: byCat.map((r) => ({ category: r.category, total_usd: Number(r.total_usd) })),
+    by_builder: byBuilder.map((r) => ({
+      builder_id: r.builder_id == null ? null : Number(r.builder_id),
+      github_login: r.github_login || null,
+      display_name: r.display_name || null,
+      total_usd: Number(r.total_usd),
+    })),
+    daily: history.map((r) => ({ day: r.day, amount_usd: Number(r.amount_usd) })),
+    daily_credits: valueHistory.map((r) => ({ day: r.day, credits: Number(r.credits) })),
+    // System 1 / task 1345 — additive dimensions (existing keys keep shape).
+    by_source: costSummaryShape.shapeBySource(bySourceRows),
+    by_tier: costSummaryShape.shapeByTier(byTierRows[0]),
+    savings: costSummaryShape.shapeSavings(savingsRows),
+    // task 572 / spend-audit F6 — cost-per-task & cost-per-credit medians.
+    cost_calibration: costSummaryShape.shapeCostCalibration(calibrationRows),
+    // task 1466 — treasury filtering. `filtered_total_usd` is the spend under
+    // the active filter (== total_usd when no filter); `filter_options` lists
+    // every source + category for the dropdowns; `applied_filters` echoes what
+    // was applied so the client can label + preselect.
+    filtered_total_usd: Number(filteredTotalRows[0].total_usd),
+    filter_options: {
+      sources: sourceOptRows.map((r) => r.source),
+      categories: categoryOptRows.map((r) => r.category),
+    },
+    applied_filters: {
+      source: filters.source || null,
+      category: filters.category || null,
+      days: daysActive ? days : null,
+    },
+  };
+}
+
+// costCalibrationSummary() — direct read of the migration-143 cost medians
+// (cost-per-task / cost-per-credit per kind + overall), shaped the same way the
+// public cost-summary exposes them. Thin SQL+shape wrapper, mirroring the
+// transparency role pms_calibration_summary plays for estimates. Returns empty
+// calibration if the function isn't deployed yet (same guard as costSummary).
+async function costCalibrationSummary() {
+  try {
+    const { rows } = await pool.query('SELECT * FROM cost_calibration_summary()');
+    return costSummaryShape.shapeCostCalibration(rows);
+  } catch (e) {
+    if (e && e.code !== '42883') throw e;
+    return costSummaryShape.shapeCostCalibration([]);
+  }
+}
+
+module.exports = {
+  getBuilderCostStatus,
+  autonomousApiSpendMtd,
+  setBuilderMonthlyBudget,
+  logCost,
+  reconcileSessionCosts,
+  costSummary,
+  costCalibrationSummary,
+};