nilometer 0.0.0-stage → 0.2.0

package/core/schema/012_unsplit_cache_writes.sql

@@ -0,0 +1,49 @@
+-- 012_unsplit_cache_writes: unsplit cache writes sum to 0, not NULL, when every request has the split.
+--
+-- In 010, SUM over cache_write_unsplit_tokens returned NULL for a model or repository whose requests
+-- all recorded the 5m/1h split, and the viewer showed "unknown" for tokens that were known to be 0.
+-- A NULL in the 5m and 1h columns still means unknown: those tokens exist, but their duration wasn't
+-- recorded. Views are recreated rather than editing 010, which existing databases have applied.
+
+DROP VIEW obs_usage_by_model;
+DROP VIEW obs_usage_by_repo;
+
+-- Observed: tokens per model per token type, with request counts and unknowns.
+CREATE VIEW obs_usage_by_model AS
+SELECT
+  model,
+  COUNT(*) AS requests,
+  SUM(input_tokens) AS input_tokens,
+  SUM(output_tokens) AS output_tokens,
+  SUM(cache_read_tokens) AS cache_read_tokens,
+  SUM(cache_write_5m_tokens) AS cache_write_5m_tokens,
+  SUM(cache_write_1h_tokens) AS cache_write_1h_tokens,
+  -- NULL here means every request recorded the 5m/1h split, so there were no unsplit writes: 0, not unknown.
+  COALESCE(SUM(cache_write_unsplit_tokens), 0) AS cache_write_unsplit_tokens,
+  COUNT(*) FILTER (WHERE dedup_key IS NULL) AS unkeyed_requests,
+  COUNT(*) FILTER (WHERE timestamp_utc IS NULL) AS requests_without_timestamp,
+  MIN(timestamp_utc) AS first_request_at_utc,
+  MAX(timestamp_utc) AS last_request_at_utc,
+  (SELECT covers_from FROM source_coverage WHERE source = 'session_logs') AS covers_from,
+  (SELECT covers_to FROM source_coverage WHERE source = 'session_logs') AS covers_to
+FROM obs_usage_events
+GROUP BY model;
+
+-- Observed: tokens per repository (git root, or the working directory outside git; NULL when no cwd) per token type.
+CREATE VIEW obs_usage_by_repo AS
+SELECT
+  repository,
+  repo_kind,
+  COUNT(*) AS requests,
+  SUM(input_tokens) AS input_tokens,
+  SUM(output_tokens) AS output_tokens,
+  SUM(cache_read_tokens) AS cache_read_tokens,
+  SUM(cache_write_5m_tokens) AS cache_write_5m_tokens,
+  SUM(cache_write_1h_tokens) AS cache_write_1h_tokens,
+  -- NULL here means every request recorded the 5m/1h split, so there were no unsplit writes: 0, not unknown.
+  COALESCE(SUM(cache_write_unsplit_tokens), 0) AS cache_write_unsplit_tokens,
+  COUNT(*) FILTER (WHERE timestamp_utc IS NULL) AS requests_without_timestamp,
+  (SELECT covers_from FROM source_coverage WHERE source = 'session_logs') AS covers_from,
+  (SELECT covers_to FROM source_coverage WHERE source = 'session_logs') AS covers_to
+FROM obs_usage_events
+GROUP BY repository, repo_kind;

package/core/schema/013_repository_revalidation.sql

@@ -0,0 +1,104 @@
+-- 013_repository_revalidation: repositories remembers how each working directory resolved and whether its root exists (D-028).
+--
+-- 005 cached one resolution per working directory forever, so a renamed or deleted folder kept its
+-- "git repository" label, and a subfolder deleted before its first ingest couldn't be placed in its
+-- repository. The table is rebuilt because SQLite can't change CHECK constraints in place, and the
+-- views that read it are recreated around the rebuild. Only 'repo' and 'not_git' rows are carried
+-- over: 'missing' rows are dropped so the next ingest resolves them again with the parent-folder
+-- lookup. Every carried row is re-checked against the filesystem on that ingest too.
+
+DROP VIEW obs_usage_by_model;
+DROP VIEW obs_usage_by_repo;
+DROP VIEW obs_usage_events;
+DROP VIEW request_repositories;
+
+CREATE TABLE repositories_013 (
+  cwd          TEXT PRIMARY KEY, -- source: cwd from log lines
+  kind         TEXT NOT NULL,    -- derived: the working directory at the last check: 'repo', 'not_git', or 'missing'
+  repo_root    TEXT,             -- derived: git's root for 'repo'; for 'missing', the last-known or nearest existing parent's root; else NULL
+  root_exists  INTEGER,          -- derived: 1 if repo_root existed at the last check, 0 if not; NULL without repo_root
+  resolved_via TEXT NOT NULL,    -- derived: 'git' (run in the directory), 'last_known', 'parent', or 'none'
+  resolved_at  TEXT NOT NULL,    -- derived: ISO-8601 UTC time of the last resolution
+  CHECK (kind IN ('repo', 'not_git', 'missing')),
+  CHECK (resolved_via IN ('git', 'last_known', 'parent', 'none')),
+  CHECK (root_exists IN (0, 1)),
+  CHECK ((repo_root IS NULL) = (root_exists IS NULL)),
+  CHECK (kind <> 'repo' OR (repo_root IS NOT NULL AND resolved_via = 'git')),
+  CHECK (kind <> 'not_git' OR (repo_root IS NULL AND resolved_via = 'git')),
+  CHECK (kind <> 'missing' OR (resolved_via = 'none') = (repo_root IS NULL)),
+  CHECK (kind = 'missing' OR resolved_via = 'git')
+) STRICT;
+
+INSERT INTO repositories_013 (cwd, kind, repo_root, root_exists, resolved_via, resolved_at)
+SELECT cwd, kind, repo_root, CASE WHEN repo_root IS NULL THEN NULL ELSE 1 END, 'git', resolved_at
+FROM repositories
+WHERE kind IN ('repo', 'not_git');
+
+DROP TABLE repositories;
+ALTER TABLE repositories_013 RENAME TO repositories;
+
+-- Observed: each request's repository and how it was attributed (D-010, D-028); a missing folder whose repository still exists counts as that repository.
+CREATE VIEW request_repositories AS
+SELECT
+  d.raw_line_id,
+  d.cwd,
+  CASE
+    WHEN r.cwd IS NULL THEN 'unresolved'
+    WHEN r.repo_root IS NOT NULL AND r.root_exists = 1 THEN 'repo'
+    WHEN r.repo_root IS NOT NULL THEN 'missing'
+    ELSE r.kind
+  END AS repo_kind,
+  COALESCE(r.repo_root, d.cwd) AS repository,
+  r.resolved_via
+FROM requests_dedup d
+LEFT JOIN repositories r ON r.cwd = d.cwd;
+
+-- Observed: every deduplicated request with the model and repository it's attributed to.
+CREATE VIEW obs_usage_events AS
+SELECT
+  d.raw_line_id, d.dedup_key, d.session_id, d.timestamp_utc, d.model,
+  r.repository, r.repo_kind,
+  d.input_tokens, d.output_tokens, d.cache_read_tokens,
+  d.cache_write_5m_tokens, d.cache_write_1h_tokens, d.cache_write_unsplit_tokens
+FROM requests_dedup d
+JOIN request_repositories r ON r.raw_line_id = d.raw_line_id;
+
+-- Observed: tokens per model per token type, with request counts and unknowns.
+CREATE VIEW obs_usage_by_model AS
+SELECT
+  model,
+  COUNT(*) AS requests,
+  SUM(input_tokens) AS input_tokens,
+  SUM(output_tokens) AS output_tokens,
+  SUM(cache_read_tokens) AS cache_read_tokens,
+  SUM(cache_write_5m_tokens) AS cache_write_5m_tokens,
+  SUM(cache_write_1h_tokens) AS cache_write_1h_tokens,
+  -- NULL here means every request recorded the 5m/1h split, so there were no unsplit writes: 0, not unknown.
+  COALESCE(SUM(cache_write_unsplit_tokens), 0) AS cache_write_unsplit_tokens,
+  COUNT(*) FILTER (WHERE dedup_key IS NULL) AS unkeyed_requests,
+  COUNT(*) FILTER (WHERE timestamp_utc IS NULL) AS requests_without_timestamp,
+  MIN(timestamp_utc) AS first_request_at_utc,
+  MAX(timestamp_utc) AS last_request_at_utc,
+  (SELECT covers_from FROM source_coverage WHERE source = 'session_logs') AS covers_from,
+  (SELECT covers_to FROM source_coverage WHERE source = 'session_logs') AS covers_to
+FROM obs_usage_events
+GROUP BY model;
+
+-- Observed: tokens per repository (git root, or the working directory outside git; NULL when no cwd) per token type.
+CREATE VIEW obs_usage_by_repo AS
+SELECT
+  repository,
+  repo_kind,
+  COUNT(*) AS requests,
+  SUM(input_tokens) AS input_tokens,
+  SUM(output_tokens) AS output_tokens,
+  SUM(cache_read_tokens) AS cache_read_tokens,
+  SUM(cache_write_5m_tokens) AS cache_write_5m_tokens,
+  SUM(cache_write_1h_tokens) AS cache_write_1h_tokens,
+  -- NULL here means every request recorded the 5m/1h split, so there were no unsplit writes: 0, not unknown.
+  COALESCE(SUM(cache_write_unsplit_tokens), 0) AS cache_write_unsplit_tokens,
+  COUNT(*) FILTER (WHERE timestamp_utc IS NULL) AS requests_without_timestamp,
+  (SELECT covers_from FROM source_coverage WHERE source = 'session_logs') AS covers_from,
+  (SELECT covers_to FROM source_coverage WHERE source = 'session_logs') AS covers_to
+FROM obs_usage_events
+GROUP BY repository, repo_kind;

package/core/schema/014_observed_readings.sql

@@ -0,0 +1,204 @@
+-- 014_observed_readings: date status line readings by the API response they came from, not by capture (D-044).
+--
+-- A payload's rate_limits are the numbers from the latest API response in that session; re-running
+-- the status line (a refresh timer, a window reset, /compact) repeats them. So a reading is observed
+-- at the latest request line in its session at or before the capture (+1 s for whole-second
+-- captures), within the window it reports, and readings of the same response collapse to one
+-- observation. Every view that ordered, dated, counted, or excluded readings by capture time is
+-- recreated here to use the observed time. Dependent views keep their column names and need no change.
+
+DROP VIEW obs_unattributed_usage;
+DROP VIEW obs_unattributed_usage_events;
+DROP VIEW window_reading_pairs;
+DROP VIEW obs_window_peak;
+DROP VIEW obs_window_headroom;
+DROP VIEW window_instances;
+DROP VIEW status_limit_groups;
+DROP VIEW source_coverage;
+DROP VIEW window_readings;
+
+-- Observed: one row per observation of a plan window: the latest capture of each API response's numbers, with when they were observed (D-024, D-044).
+CREATE VIEW window_readings AS
+WITH readings AS (
+  SELECT
+    w.raw_line_id, w.window, w.resets_at, w.used_percentage, s.session_id, s.captured_at_s,
+    -- Only a request inside this window can have produced numbers for it; +1 s covers whole-second captures.
+    (SELECT MAX(p.timestamp_utc) FROM parsed_lines p
+       WHERE p.session_id = s.session_id AND p.class = 'request'
+         AND p.timestamp_utc > strftime('%Y-%m-%dT%H:%M:%fZ',
+               w.resets_at - CASE w.window WHEN 'five_hour' THEN 5 * 3600 ELSE 7 * 86400 END, 'unixepoch')
+         AND p.timestamp_utc < strftime('%Y-%m-%dT%H:%M:%fZ', s.captured_at_s + 1, 'unixepoch')) AS last_request_utc
+  FROM rate_limit_windows w
+  JOIN status_readings s ON s.raw_line_id = w.raw_line_id
+  WHERE s.status = 'ok' AND w.validity = 'valid' AND w.window IN ('five_hour', 'seven_day')
+),
+observed AS (
+  SELECT
+    r.*,
+    COALESCE(r.last_request_utc, strftime('%Y-%m-%dT%H:%M:%fZ', r.captured_at_s, 'unixepoch')) AS observed_at_utc,
+    ROW_NUMBER() OVER (
+      -- A capture-time fallback is its own observation: NULL request times must not merge.
+      PARTITION BY r.window, r.resets_at, r.session_id, r.last_request_utc,
+                   CASE WHEN r.last_request_utc IS NULL THEN r.raw_line_id END
+      ORDER BY r.captured_at_s DESC, r.raw_line_id DESC
+    ) AS copy_rank
+  FROM readings r
+)
+SELECT
+  raw_line_id,
+  window,
+  resets_at,
+  strftime('%Y-%m-%dT%H:%M:%fZ', resets_at, 'unixepoch') AS reset_at_utc,
+  used_percentage,
+  session_id,
+  captured_at_s,
+  strftime('%Y-%m-%dT%H:%M:%fZ', captured_at_s, 'unixepoch') AS captured_at_utc,
+  observed_at_utc,
+  unixepoch(observed_at_utc, 'subsec') AS observed_at_s,
+  CASE WHEN last_request_utc IS NULL THEN 'capture' ELSE 'last_request' END AS observed_via,
+  unixepoch(observed_at_utc, 'subsec') > resets_at AS after_reset
+FROM observed
+WHERE copy_rank = 1;
+
+-- Observed: the time span each source's data covers, from the data present (README principle 5); status line spans observations (D-044).
+CREATE VIEW source_coverage AS
+SELECT 'session_logs' AS source, MIN(timestamp_utc) AS covers_from, MAX(timestamp_utc) AS covers_to
+FROM parsed_lines
+WHERE timestamp_utc IS NOT NULL
+UNION ALL
+SELECT 'status_line', MIN(observed_at_utc), MAX(observed_at_utc)
+FROM window_readings;
+
+-- Observed: status line windows observed at 100%, one row per (window, resets_at), from init onward (D-023, D-044).
+CREATE VIEW status_limit_groups AS
+WITH at_limit AS (
+  SELECT
+    r.raw_line_id, r.window, r.resets_at, r.session_id, r.observed_at_utc,
+    ROW_NUMBER() OVER (PARTITION BY r.window, r.resets_at ORDER BY r.observed_at_s, r.raw_line_id) AS reading_rank,
+    COUNT(*) OVER (PARTITION BY r.window, r.resets_at) AS readings_at_limit
+  FROM window_readings r
+  WHERE r.used_percentage >= 100
+)
+SELECT
+  raw_line_id, window, resets_at, session_id, readings_at_limit,
+  observed_at_utc AS first_at_utc,
+  strftime('%Y-%m-%dT%H:%M:%fZ', resets_at, 'unixepoch') AS reset_at_utc,
+  CASE window WHEN 'five_hour' THEN 5 * 3600 ELSE 7 * 86400 END AS window_seconds
+FROM at_limit
+WHERE reading_rank = 1;
+
+-- Observed: per window instance, observation counts, whether it is still open, and the status line coverage.
+CREATE VIEW window_instances AS
+SELECT
+  window,
+  resets_at,
+  reset_at_utc,
+  COUNT(*) FILTER (WHERE after_reset = 0) AS readings,
+  COUNT(*) FILTER (WHERE after_reset = 1) AS readings_after_reset,
+  MIN(observed_at_utc) FILTER (WHERE after_reset = 0) AS first_reading_at_utc,
+  -- Open: no status line reading at all was captured at or after this reset, so later usage may still come.
+  resets_at > (SELECT MAX(captured_at_s) FROM status_readings WHERE status = 'ok') AS window_open,
+  (SELECT covers_from FROM source_coverage WHERE source = 'status_line') AS covers_from,
+  (SELECT covers_to FROM source_coverage WHERE source = 'status_line') AS covers_to
+FROM window_readings
+GROUP BY window, resets_at;
+
+-- Observed: last observed usage before each window's reset, with when it was observed (a lower bound on final usage; D-024, D-044).
+CREATE VIEW obs_window_headroom AS
+WITH ranked AS (
+  SELECT
+    r.*,
+    ROW_NUMBER() OVER (PARTITION BY r.window, r.resets_at ORDER BY r.observed_at_s DESC, r.raw_line_id DESC) AS latest_rank
+  FROM window_readings r
+  WHERE r.after_reset = 0
+)
+SELECT
+  i.window,
+  i.reset_at_utc,
+  r.used_percentage AS last_used_percentage,
+  r.observed_at_utc AS last_reading_at_utc,
+  r.raw_line_id AS last_reading_raw_line_id,
+  i.readings,
+  i.readings_after_reset,
+  i.first_reading_at_utc,
+  i.window_open,
+  i.covers_from,
+  i.covers_to
+FROM window_instances i
+LEFT JOIN ranked r ON r.window = i.window AND r.resets_at = i.resets_at AND r.latest_rank = 1;
+
+-- Observed: highest observed usage in each window, with the observation it came from (D-044).
+CREATE VIEW obs_window_peak AS
+WITH ranked AS (
+  SELECT
+    r.*,
+    ROW_NUMBER() OVER (PARTITION BY r.window, r.resets_at ORDER BY r.used_percentage DESC, r.observed_at_s, r.raw_line_id) AS peak_rank
+  FROM window_readings r
+  WHERE r.after_reset = 0
+)
+SELECT
+  i.window,
+  i.reset_at_utc,
+  r.used_percentage AS peak_used_percentage,
+  r.observed_at_utc AS peak_reading_at_utc,
+  r.raw_line_id AS peak_reading_raw_line_id,
+  i.readings,
+  i.readings_after_reset,
+  i.window_open,
+  i.covers_from,
+  i.covers_to
+FROM window_instances i
+LEFT JOIN ranked r ON r.window = i.window AND r.resets_at = i.resets_at AND r.peak_rank = 1;
+
+-- Observed: consecutive observations of one window, the change between them, and Claude Code requests in between (D-025, D-044).
+CREATE VIEW window_reading_pairs AS
+WITH ordered AS (
+  SELECT
+    r.*,
+    LAG(r.raw_line_id) OVER w AS previous_raw_line_id,
+    LAG(r.used_percentage) OVER w AS previous_used_percentage,
+    LAG(r.observed_at_utc) OVER w AS previous_observed_at_utc
+  FROM window_readings r
+  WHERE r.after_reset = 0
+  WINDOW w AS (PARTITION BY r.window, r.resets_at ORDER BY r.observed_at_s, r.raw_line_id)
+)
+SELECT
+  o.window,
+  o.reset_at_utc,
+  o.previous_raw_line_id,
+  o.raw_line_id,
+  o.previous_observed_at_utc,
+  o.observed_at_utc,
+  o.used_percentage - o.previous_used_percentage AS change_percentage_points,
+  -- Include the previous observation's instant, and one second past this one (whole-second captures).
+  (SELECT COUNT(*) FROM requests_dedup d
+     WHERE d.timestamp_utc >= o.previous_observed_at_utc
+       AND d.timestamp_utc < strftime('%Y-%m-%dT%H:%M:%fZ', o.observed_at_s + 1, 'unixepoch')) AS requests_between
+FROM ordered o
+WHERE o.previous_raw_line_id IS NOT NULL;
+
+-- Observed: observation pairs whose usage rose with no Claude Code request in between (D-025; see D-044 on why real data rarely has any).
+CREATE VIEW obs_unattributed_usage_events AS
+SELECT * FROM window_reading_pairs
+WHERE requests_between = 0 AND change_percentage_points > 0;
+
+-- Observed: per window, percentage points consumed with no Claude Code request in between (a lower bound), beside pair counts.
+CREATE VIEW obs_unattributed_usage AS
+SELECT
+  i.window,
+  i.reset_at_utc,
+  i.readings,
+  (SELECT COUNT(*) FROM window_reading_pairs p WHERE p.window = i.window AND p.reset_at_utc = i.reset_at_utc) AS pairs,
+  (SELECT COUNT(*) FROM window_reading_pairs p
+     WHERE p.window = i.window AND p.reset_at_utc = i.reset_at_utc AND p.requests_between = 0) AS pairs_without_requests,
+  (SELECT COUNT(*) FROM window_reading_pairs p
+     WHERE p.window = i.window AND p.reset_at_utc = i.reset_at_utc AND p.change_percentage_points < 0) AS decreasing_pairs,
+  -- Fewer than two observations means no pair was observed: unknown, not 0.
+  CASE WHEN i.readings >= 2 THEN
+    (SELECT TOTAL(e.change_percentage_points) FROM obs_unattributed_usage_events e
+       WHERE e.window = i.window AND e.reset_at_utc = i.reset_at_utc)
+  END AS unattributed_percentage_points,
+  i.window_open,
+  i.covers_from,
+  i.covers_to
+FROM window_instances i;

package/core/schema/015_response_observations.sql

@@ -0,0 +1,127 @@
+-- 015_response_observations: one status line observation per API response, dated by its first log line (D-045).
+--
+-- 014 keyed observations by the latest request line, but one response is several streaming snapshot
+-- lines with their own timestamps (D-001), so readings taken mid-stream became separate observations
+-- of one response. Here the key is the response (its D-001 dedup key, or the line itself when
+-- unkeyed), dated by the response's first line, and reading pairs count distinct responses with any
+-- line in the interval. Dependent views keep their column names.
+
+DROP VIEW obs_unattributed_usage;
+DROP VIEW obs_unattributed_usage_events;
+DROP VIEW window_reading_pairs;
+DROP VIEW window_readings;
+
+-- Observed: one row per observation of a plan window: the latest capture of each API response's numbers, dated by that response's first line (D-044, D-045).
+CREATE VIEW window_readings AS
+WITH readings AS (
+  SELECT
+    w.raw_line_id, w.window, w.resets_at, w.used_percentage, s.session_id, s.captured_at_s,
+    -- The response behind the latest request line in this session, inside this window, at or before
+    -- the capture (+1 s for whole-second captures). An unkeyed line is its own response.
+    (SELECT COALESCE(rq.dedup_key, 'line:' || rq.raw_line_id)
+       FROM parsed_lines p
+       JOIN requests rq ON rq.raw_line_id = p.raw_line_id
+       WHERE p.session_id = s.session_id AND p.class = 'request'
+         AND p.timestamp_utc > strftime('%Y-%m-%dT%H:%M:%fZ',
+               w.resets_at - CASE w.window WHEN 'five_hour' THEN 5 * 3600 ELSE 7 * 86400 END, 'unixepoch')
+         AND p.timestamp_utc < strftime('%Y-%m-%dT%H:%M:%fZ', s.captured_at_s + 1, 'unixepoch')
+       ORDER BY p.timestamp_utc DESC, p.raw_line_id DESC
+       LIMIT 1) AS response_key
+  FROM rate_limit_windows w
+  JOIN status_readings s ON s.raw_line_id = w.raw_line_id
+  WHERE s.status = 'ok' AND w.validity = 'valid' AND w.window IN ('five_hour', 'seven_day')
+),
+dated AS (
+  SELECT
+    r.*,
+    -- The response's first line: when its numbers first existed.
+    (SELECT MIN(p.timestamp_utc)
+       FROM requests rq
+       JOIN parsed_lines p ON p.raw_line_id = rq.raw_line_id
+       WHERE (r.response_key LIKE 'line:%' AND rq.raw_line_id = CAST(substr(r.response_key, 6) AS INTEGER))
+          OR rq.dedup_key = r.response_key) AS response_first_utc
+  FROM readings r
+),
+observed AS (
+  SELECT
+    d.*,
+    COALESCE(d.response_first_utc, strftime('%Y-%m-%dT%H:%M:%fZ', d.captured_at_s, 'unixepoch')) AS observed_at_utc,
+    ROW_NUMBER() OVER (
+      -- A capture-time fallback is its own observation: NULL response keys must not merge.
+      PARTITION BY d.window, d.resets_at, d.session_id, d.response_key,
+                   CASE WHEN d.response_key IS NULL THEN d.raw_line_id END
+      ORDER BY d.captured_at_s DESC, d.raw_line_id DESC
+    ) AS copy_rank
+  FROM dated d
+)
+SELECT
+  raw_line_id,
+  window,
+  resets_at,
+  strftime('%Y-%m-%dT%H:%M:%fZ', resets_at, 'unixepoch') AS reset_at_utc,
+  used_percentage,
+  session_id,
+  captured_at_s,
+  strftime('%Y-%m-%dT%H:%M:%fZ', captured_at_s, 'unixepoch') AS captured_at_utc,
+  observed_at_utc,
+  unixepoch(observed_at_utc, 'subsec') AS observed_at_s,
+  CASE WHEN response_key IS NULL THEN 'capture' ELSE 'last_request' END AS observed_via,
+  unixepoch(observed_at_utc, 'subsec') > resets_at AS after_reset
+FROM observed
+WHERE copy_rank = 1;
+
+-- Observed: consecutive observations of one window, the change between them, and Claude Code responses in between (D-025, D-045).
+CREATE VIEW window_reading_pairs AS
+WITH ordered AS (
+  SELECT
+    r.*,
+    LAG(r.raw_line_id) OVER w AS previous_raw_line_id,
+    LAG(r.used_percentage) OVER w AS previous_used_percentage,
+    LAG(r.observed_at_utc) OVER w AS previous_observed_at_utc
+  FROM window_readings r
+  WHERE r.after_reset = 0
+  WINDOW w AS (PARTITION BY r.window, r.resets_at ORDER BY r.observed_at_s, r.raw_line_id)
+)
+SELECT
+  o.window,
+  o.reset_at_utc,
+  o.previous_raw_line_id,
+  o.raw_line_id,
+  o.previous_observed_at_utc,
+  o.observed_at_utc,
+  o.used_percentage - o.previous_used_percentage AS change_percentage_points,
+  -- Distinct responses with any line from the previous observation up to one second past this one.
+  (SELECT COUNT(DISTINCT COALESCE(rq.dedup_key, 'line:' || rq.raw_line_id))
+     FROM parsed_lines p
+     JOIN requests rq ON rq.raw_line_id = p.raw_line_id
+     WHERE p.class = 'request'
+       AND p.timestamp_utc >= o.previous_observed_at_utc
+       AND p.timestamp_utc < strftime('%Y-%m-%dT%H:%M:%fZ', o.observed_at_s + 1, 'unixepoch')) AS requests_between
+FROM ordered o
+WHERE o.previous_raw_line_id IS NOT NULL;
+
+-- Observed: observation pairs whose usage rose with no Claude Code request in between (D-025; see D-044 on why real data rarely has any).
+CREATE VIEW obs_unattributed_usage_events AS
+SELECT * FROM window_reading_pairs
+WHERE requests_between = 0 AND change_percentage_points > 0;
+
+-- Observed: per window, percentage points consumed with no Claude Code request in between (a lower bound), beside pair counts.
+CREATE VIEW obs_unattributed_usage AS
+SELECT
+  i.window,
+  i.reset_at_utc,
+  i.readings,
+  (SELECT COUNT(*) FROM window_reading_pairs p WHERE p.window = i.window AND p.reset_at_utc = i.reset_at_utc) AS pairs,
+  (SELECT COUNT(*) FROM window_reading_pairs p
+     WHERE p.window = i.window AND p.reset_at_utc = i.reset_at_utc AND p.requests_between = 0) AS pairs_without_requests,
+  (SELECT COUNT(*) FROM window_reading_pairs p
+     WHERE p.window = i.window AND p.reset_at_utc = i.reset_at_utc AND p.change_percentage_points < 0) AS decreasing_pairs,
+  -- Fewer than two observations means no pair was observed: unknown, not 0.
+  CASE WHEN i.readings >= 2 THEN
+    (SELECT TOTAL(e.change_percentage_points) FROM obs_unattributed_usage_events e
+       WHERE e.window = i.window AND e.reset_at_utc = i.reset_at_utc)
+  END AS unattributed_percentage_points,
+  i.window_open,
+  i.covers_from,
+  i.covers_to
+FROM window_instances i;

package/core/schema/016_windows_repository_paths.sql

@@ -0,0 +1,41 @@
+-- 016_windows_repository_paths: a Windows repository path is written one way, whatever its source (D-049).
+--
+-- Observed on a Windows PC (2026-09-17, counts and structure only): Claude Code logged working
+-- directories with `\` separators and with the drive letter both as `C:` and `c:`, while git returns
+-- repository roots with `/` separators. For a folder that still exists, git resolved both spellings to
+-- one root. A deleted folder with no known root is its own repository, so one such folder appeared as
+-- two rows. Here a repository that starts with a drive letter gets an uppercase letter and `/`
+-- separators. Windows paths are case-insensitive for the drive, and a POSIX repository is an absolute
+-- path starting with `/`, so no POSIX value changes. The stored cwd and repo_root keep their source
+-- spelling; only the attributed repository is normalized.
+
+DROP VIEW request_repositories;
+
+-- Observed: each request's repository and how it was attributed (D-010, D-028); a missing folder whose repository still exists counts as that repository; a Windows path is written with an uppercase drive letter and `/` separators (D-049).
+CREATE VIEW request_repositories AS
+WITH attributed AS (
+  SELECT
+    d.raw_line_id,
+    d.cwd,
+    CASE
+      WHEN r.cwd IS NULL THEN 'unresolved'
+      WHEN r.repo_root IS NOT NULL AND r.root_exists = 1 THEN 'repo'
+      WHEN r.repo_root IS NOT NULL THEN 'missing'
+      ELSE r.kind
+    END AS repo_kind,
+    COALESCE(r.repo_root, d.cwd) AS repository,
+    r.resolved_via
+  FROM requests_dedup d
+  LEFT JOIN repositories r ON r.cwd = d.cwd
+)
+SELECT
+  raw_line_id,
+  cwd,
+  repo_kind,
+  CASE
+    WHEN repository GLOB '[A-Za-z]:[\/]*'
+    THEN upper(substr(repository, 1, 1)) || replace(substr(repository, 2), char(92), '/')
+    ELSE repository
+  END AS repository,
+  resolved_via
+FROM attributed;

package/core/schema/017_unattributed_usage_once.sql

@@ -0,0 +1,47 @@
+-- 017_unattributed_usage_once: compute the reading pairs once per window instead of four times.
+--
+-- obs_unattributed_usage asked window_reading_pairs four separate correlated questions per window
+-- instance, and window_reading_pairs answers each one by running its own correlated subquery over
+-- parsed_lines for every pair. Reading the view took about 8.9 seconds on a real database, 55% of
+-- the whole report, to return one row per window. The numbers were right; they were just computed
+-- many times over.
+--
+-- The rewrite aggregates the pairs once, grouped by window instance, and joins that to the
+-- instances. No column, value, or row changes: a window with no pairs reported zeros through the
+-- subqueries and reports them through COALESCE here, and TOTAL() over no rows is 0.0, which the
+-- COALESCE reproduces. Source: no API field; this view is built from status line readings (D-025,
+-- D-044).
+DROP VIEW obs_unattributed_usage;
+
+-- Observed: per window, percentage points consumed with no Claude Code request in between (a lower bound), beside pair counts.
+CREATE VIEW obs_unattributed_usage AS
+WITH pair_totals AS (
+  SELECT
+    p.window,
+    p.reset_at_utc,
+    COUNT(*) AS pairs,
+    COUNT(*) FILTER (WHERE p.requests_between = 0) AS pairs_without_requests,
+    COUNT(*) FILTER (WHERE p.change_percentage_points < 0) AS decreasing_pairs,
+    -- The same rows obs_unattributed_usage_events selects: rose, with no request in between.
+    TOTAL(
+      CASE WHEN p.requests_between = 0 AND p.change_percentage_points > 0
+        THEN p.change_percentage_points END
+    ) AS unattributed_percentage_points
+  FROM window_reading_pairs p
+  GROUP BY p.window, p.reset_at_utc
+)
+SELECT
+  i.window,
+  i.reset_at_utc,
+  i.readings,
+  COALESCE(t.pairs, 0) AS pairs,
+  COALESCE(t.pairs_without_requests, 0) AS pairs_without_requests,
+  COALESCE(t.decreasing_pairs, 0) AS decreasing_pairs,
+  -- Fewer than two observations means no pair was observed: unknown, not 0.
+  CASE WHEN i.readings >= 2 THEN COALESCE(t.unattributed_percentage_points, 0.0) END
+    AS unattributed_percentage_points,
+  i.window_open,
+  i.covers_from,
+  i.covers_to
+FROM window_instances i
+LEFT JOIN pair_totals t ON t.window = i.window AND t.reset_at_utc = i.reset_at_utc;

package/core/schema/018_request_time_index.sql

@@ -0,0 +1,21 @@
+-- 018_request_time_index: index request lines by time, so counting the responses between two status
+-- line readings is a range scan rather than a table scan.
+--
+-- window_reading_pairs counts distinct responses in an interval, once per pair. That subquery
+-- filters parsed_lines by class and a timestamp range with no session, and every existing index
+-- leads with session_id, so each of the hundreds of pairs scanned the whole table.
+--
+-- The index is PARTIAL on purpose. A plain index on (class, timestamp_utc) fixed this query but
+-- tempted the planner into using it for the window views, which filter by session and time. Measured
+-- on a real database, as medians of three runs:
+--     view                      no index   full index   partial index
+--     obs_unattributed_usage        9347         1794            1236
+--     obs_lockout_time              2296         3391            2313
+--     obs_sessions_not_resumed      1157         1740            1167
+--     every report view (total)    17281        13633            9285
+-- A partial index can only be used by a query carrying the same WHERE clause, so it serves this
+-- subquery and is invisible to the rest. Every shape returned identical rows; only the time changed.
+--
+-- The cost is one small index and slightly slower inserts during ingest, which writes each line once
+-- and reads these views on every report.
+CREATE INDEX parsed_lines_request_time ON parsed_lines (timestamp_utc) WHERE class = 'request';

package/core/schema/019_dedup_earliest_of_equals.sql

@@ -0,0 +1,61 @@
+-- 019_dedup_earliest_of_equals: when the dedup rule cannot choose, take the earliest line.
+--
+-- One API response is written as several lines and the largest output_tokens wins (D-001). When
+-- every line carries the SAME counts the rule selects nothing, and the tie-breaks that followed it
+-- ended on `line_number DESC` — the last line — which decided nothing except, at a day boundary,
+-- which day the response landed on. Observed on a real log set: one response written three times
+-- across 23:59:57 to 00:00:01, counted on the later day here and the earlier one by ccusage.
+--
+-- The rule this project already states for the same problem is D-045: collapse duplicates of one
+-- response and date it by its first line, "when its numbers first existed". That is applied here.
+-- Nothing changes where output_tokens grow, because the largest still wins before this is reached.
+-- Source: no API field; this is a choice among lines the API produced (D-001, D-045, D-065).
+DROP VIEW requests_dedup;
+
+-- Observed: one row per distinct API response (winners) plus every unkeyed request line.
+CREATE VIEW requests_dedup AS
+WITH ranked AS (
+  SELECT
+    r.raw_line_id,
+    r.dedup_key,
+    r.message_id,
+    r.request_id,
+    r.model,
+    r.input_tokens,
+    r.output_tokens,
+    r.cache_read_tokens,
+    r.cache_write_5m_tokens,
+    r.cache_write_1h_tokens,
+    r.cache_write_unsplit_tokens,
+    r.speed,
+    p.session_id,
+    p.timestamp_raw,
+    p.timestamp_utc,
+    p.cwd,
+    p.is_sidechain,
+    f.root,
+    f.relative_path,
+    l.first_run_id,
+    l.line_number,
+    ROW_NUMBER() OVER (
+      PARTITION BY r.dedup_key
+      ORDER BY r.output_tokens DESC, p.is_sidechain ASC,
+               -- Among lines tied on output, the earliest: when those numbers first existed
+               -- (D-065, the same rule D-045 applies to readings). NULLs last, so an unparsed
+               -- timestamp never wins.
+               (p.timestamp_utc IS NULL) ASC, p.timestamp_utc ASC,
+               f.root DESC, f.relative_path DESC, l.first_run_id DESC, l.line_number DESC
+    ) AS candidate_rank
+  FROM requests r
+  JOIN parsed_lines p ON p.raw_line_id = r.raw_line_id
+  JOIN raw_lines l ON l.id = r.raw_line_id
+  JOIN source_files f ON f.id = l.source_file_id
+)
+SELECT
+  raw_line_id, dedup_key, message_id, request_id, model,
+  input_tokens, output_tokens, cache_read_tokens,
+  cache_write_5m_tokens, cache_write_1h_tokens, cache_write_unsplit_tokens, speed,
+  session_id, timestamp_raw, timestamp_utc, cwd, is_sidechain,
+  root, relative_path, first_run_id, line_number
+FROM ranked
+WHERE dedup_key IS NULL OR candidate_rank = 1;