nilometer 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +68 -0
- package/LICENSE +21 -0
- package/README.md +464 -2
- package/core/pricing/prices.json +348 -0
- package/core/schema/001_ingestion.sql +57 -0
- package/core/schema/002_line_count.sql +7 -0
- package/core/schema/003_derived.sql +70 -0
- package/core/schema/004_requests_dedup.sql +50 -0
- package/core/schema/005_repositories.sql +25 -0
- package/core/schema/006_status_readings.sql +33 -0
- package/core/schema/007_pricing.sql +95 -0
- package/core/schema/008_interruptions.sql +263 -0
- package/core/schema/009_headroom.sql +84 -0
- package/core/schema/010_attribution.sql +106 -0
- package/core/schema/011_projections.sql +83 -0
- package/core/schema/012_unsplit_cache_writes.sql +49 -0
- package/core/schema/013_repository_revalidation.sql +104 -0
- package/core/schema/014_observed_readings.sql +204 -0
- package/core/schema/015_response_observations.sql +127 -0
- package/core/schema/016_windows_repository_paths.sql +41 -0
- package/core/schema/017_unattributed_usage_once.sql +47 -0
- package/core/schema/018_request_time_index.sql +21 -0
- package/core/schema/019_dedup_earliest_of_equals.sql +61 -0
- package/core/schema/020_limit_quota_fields.sql +131 -0
- package/core/schema/021_plan_periods.sql +121 -0
- package/core/schema/022_attachment_walk.sql +223 -0
- package/core/schema/023_auto_continuations.sql +129 -0
- package/core/schema/024_counted_limit_groups.sql +29 -0
- package/core/schema/025_model_fallback.sql +77 -0
- package/core/schema/026_changed_readings.sql +106 -0
- package/core/schema/027_weekly_crossings.sql +32 -0
- package/core/schema/028_stale_readings.sql +196 -0
- package/core/schema/029_lockout_sessions.sql +68 -0
- package/core/schema/030_repository_aliases.sql +81 -0
- package/core/schema/031_cte_cursor_budget.sql +153 -0
- package/core/schema/032_stored_views.sql +233 -0
- package/dist/cli/main.cli.js +39 -0
- package/dist/cli/program.js +881 -0
- package/dist/core/db/database.js +182 -0
- package/dist/core/db/fingerprint.js +71 -0
- package/dist/core/ingest/attribution.js +131 -0
- package/dist/core/ingest/classify.js +382 -0
- package/dist/core/ingest/command.js +95 -0
- package/dist/core/ingest/derive.js +219 -0
- package/dist/core/ingest/discover.js +126 -0
- package/dist/core/ingest/ingest.js +160 -0
- package/dist/core/ingest/reader.js +89 -0
- package/dist/core/ingest/reset-time.js +97 -0
- package/dist/core/ingest/results.js +201 -0
- package/dist/core/ingest/spool.js +165 -0
- package/dist/core/ingest/store.js +99 -0
- package/dist/core/ingest/stored-views.js +65 -0
- package/dist/core/install/install.js +185 -0
- package/dist/core/install/locations.js +89 -0
- package/dist/core/install/private-files.js +259 -0
- package/dist/core/install/record.js +103 -0
- package/dist/core/plans/plan-prices.js +128 -0
- package/dist/core/plans/repository-aliases.js +136 -0
- package/dist/core/pricing/prices.js +147 -0
- package/dist/core/settings/settings-file.js +215 -0
- package/dist/core/settings/statusline.js +240 -0
- package/dist/core/util/commands.js +91 -0
- package/dist/core/util/node-version.js +58 -0
- package/dist/core/util/open-page.js +69 -0
- package/dist/core/util/paths.js +36 -0
- package/dist/core/verify/compare.js +77 -0
- package/dist/core/verify/verify.js +220 -0
- package/dist/core/view/server.js +142 -0
- package/dist/viewer/explain.js +574 -0
- package/dist/viewer/format.js +150 -0
- package/dist/viewer/html-charts.js +1292 -0
- package/dist/viewer/html-contract.js +2 -0
- package/dist/viewer/html-data.js +718 -0
- package/dist/viewer/html.js +1717 -0
- package/dist/viewer/queries.js +128 -0
- package/dist/viewer/render.js +602 -0
- package/dist/viewer/report.js +120 -0
- package/dist/viewer/save.js +110 -0
- package/docs/requirements.md +113 -0
- package/hooks/statusline.sh +104 -0
- package/package.json +61 -4
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
-- 031_cte_cursor_budget: the lockout views read each costly view once, so they prepare on SQLite 3.54 (D-094, step G5.2).
|
|
2
|
+
--
|
|
3
|
+
-- SQLite 3.54.0 refuses a statement once a CTE's cursor number passes SQLITE_LIMIT_EXPR_DEPTH (1,000), with
|
|
4
|
+
-- "VIEWs and/or subqueries nested too deep". Every reference to a view expands it again, cursors and all, so a
|
|
5
|
+
-- view read four times costs four times its cursors. obs_lockout_intervals read the lockout hits four times
|
|
6
|
+
-- (593 by that measure), and obs_lockout_time and obs_interruptions_by_plan read it twice more: over the limit.
|
|
7
|
+
-- Node 26 bundles 3.54.0. Same rows, same columns, same order; only how often each view is read changes.
|
|
8
|
+
|
|
9
|
+
DROP VIEW obs_lockout_intervals;
|
|
10
|
+
|
|
11
|
+
-- Observed: elapsed lockout intervals (limit hit to reset, overlapping hits merged), with the next request after each reset, logged auto-continuations, and the sessions and refused requests in each (D-023, D-077, D-090).
|
|
12
|
+
CREATE VIEW obs_lockout_intervals AS
|
|
13
|
+
WITH continuation_counts AS (
|
|
14
|
+
-- Logged auto-continuations per limit hit, counted before the join so a hit stays one row.
|
|
15
|
+
SELECT hit_raw_line_id, COUNT(*) AS marks, MIN(continued_at_utc) AS first_continued_at_utc
|
|
16
|
+
FROM auto_continuation_marks
|
|
17
|
+
WHERE hit_raw_line_id IS NOT NULL
|
|
18
|
+
GROUP BY hit_raw_line_id
|
|
19
|
+
),
|
|
20
|
+
grouped AS (
|
|
21
|
+
-- One pass over the lockout hits: each is one row (its raw line), and every join below is one-to-one.
|
|
22
|
+
SELECT
|
|
23
|
+
h.interval_number,
|
|
24
|
+
MIN(h.start_utc) AS locked_from_utc,
|
|
25
|
+
MAX(h.end_utc) AS locked_until_utc,
|
|
26
|
+
COUNT(*) AS hits,
|
|
27
|
+
-- A logged hit names its session; a status-line-only hit is a spool reading, which names one too.
|
|
28
|
+
COUNT(DISTINCT COALESCE(p.session_id, s.session_id)) AS sessions,
|
|
29
|
+
-- Hits from the session logs: each is a request the limit refused.
|
|
30
|
+
COUNT(p.raw_line_id) AS refused_requests,
|
|
31
|
+
SUM(m.marks) AS auto_continuations,
|
|
32
|
+
MIN(m.first_continued_at_utc) AS first_continued_at_utc
|
|
33
|
+
FROM obs_lockout_interval_hits h
|
|
34
|
+
LEFT JOIN parsed_lines p ON p.raw_line_id = h.raw_line_id
|
|
35
|
+
LEFT JOIN status_readings s ON s.raw_line_id = h.raw_line_id
|
|
36
|
+
LEFT JOIN continuation_counts m ON m.hit_raw_line_id = h.raw_line_id
|
|
37
|
+
GROUP BY h.interval_number
|
|
38
|
+
),
|
|
39
|
+
with_next AS (
|
|
40
|
+
SELECT
|
|
41
|
+
g.*,
|
|
42
|
+
(SELECT MIN(d.timestamp_utc) FROM requests_dedup d WHERE d.timestamp_utc >= g.locked_until_utc) AS next_request_at_utc
|
|
43
|
+
FROM grouped g
|
|
44
|
+
)
|
|
45
|
+
SELECT
|
|
46
|
+
w.interval_number,
|
|
47
|
+
w.locked_from_utc,
|
|
48
|
+
w.locked_until_utc,
|
|
49
|
+
w.hits,
|
|
50
|
+
w.next_request_at_utc,
|
|
51
|
+
(round(unixepoch(w.locked_until_utc, 'subsec') * 1000) - round(unixepoch(w.locked_from_utc, 'subsec') * 1000)) / 1000.0 AS lockout_seconds,
|
|
52
|
+
(round(unixepoch(w.next_request_at_utc, 'subsec') * 1000) - round(unixepoch(w.locked_until_utc, 'subsec') * 1000)) / 1000.0 AS reset_to_next_request_seconds,
|
|
53
|
+
w.locked_until_utc > (SELECT covers_to FROM source_coverage WHERE source = 'session_logs') AS reset_after_coverage,
|
|
54
|
+
-- How many sessions Claude Code resumed by itself after this reset, as logged; 0 when none.
|
|
55
|
+
COALESCE(w.auto_continuations, 0) AS auto_continuations,
|
|
56
|
+
-- The earliest of those, seconds after the reset; NULL when none. The reset is one time per lockout, so the
|
|
57
|
+
-- earliest continuation gives the smallest difference.
|
|
58
|
+
(round(unixepoch(w.first_continued_at_utc, 'subsec') * 1000) - round(unixepoch(w.locked_until_utc, 'subsec') * 1000)) / 1000.0
|
|
59
|
+
AS first_auto_continuation_seconds,
|
|
60
|
+
-- Distinct sessions with a hit in this lockout (D-090).
|
|
61
|
+
w.sessions,
|
|
62
|
+
-- Its logged hits, each a refused request; the rest of its hits are status line readings at 100% (D-023, D-090).
|
|
63
|
+
w.refused_requests
|
|
64
|
+
FROM with_next w;
|
|
65
|
+
|
|
66
|
+
DROP VIEW obs_lockout_time;
|
|
67
|
+
|
|
68
|
+
-- Observed: total elapsed lockout time (hit → reset, overlaps counted once), beside hits whose reset is unknown.
|
|
69
|
+
CREATE VIEW obs_lockout_time AS
|
|
70
|
+
SELECT
|
|
71
|
+
COUNT(*) AS intervals,
|
|
72
|
+
TOTAL(lockout_seconds) AS lockout_seconds,
|
|
73
|
+
-- Every hit with a known reset is in exactly one interval, so the intervals' hits add up to them.
|
|
74
|
+
COALESCE(SUM(hits), 0) AS hits_with_known_reset,
|
|
75
|
+
(SELECT COUNT(*) FROM obs_limit_hits_events WHERE reset_at_utc IS NULL OR hit_at_utc IS NULL) AS hits_with_unknown_reset,
|
|
76
|
+
(SELECT covers_from FROM source_coverage WHERE source = 'session_logs') AS covers_from,
|
|
77
|
+
(SELECT covers_to FROM source_coverage WHERE source = 'session_logs') AS covers_to
|
|
78
|
+
FROM obs_lockout_intervals;
|
|
79
|
+
|
|
80
|
+
DROP VIEW obs_interruptions_by_plan;
|
|
81
|
+
|
|
82
|
+
-- Observed: limit hits, lockouts, and elapsed lockout time per plan period, each with its own span (D-075).
|
|
83
|
+
CREATE VIEW obs_interruptions_by_plan AS
|
|
84
|
+
WITH periods AS (
|
|
85
|
+
SELECT 'plan' AS period_kind, from_month, until_month, plan_name FROM plan_periods
|
|
86
|
+
UNION ALL
|
|
87
|
+
-- Before the first entered plan; with no plan entered, the whole history.
|
|
88
|
+
SELECT 'none', NULL, (SELECT MIN(from_month) FROM plan_periods), NULL
|
|
89
|
+
UNION ALL
|
|
90
|
+
-- Hits with no time: counted, so the rows add up to obs_limit_hits, but in no period.
|
|
91
|
+
SELECT 'unknown', NULL, NULL, NULL
|
|
92
|
+
),
|
|
93
|
+
-- Materialized: the hit view is among the costliest on real data (D-059), and two CTEs read it.
|
|
94
|
+
hits AS MATERIALIZED (
|
|
95
|
+
SELECT period_kind, from_month, window, hit_at_utc FROM obs_limit_hits_by_plan_events
|
|
96
|
+
),
|
|
97
|
+
-- Counted per period once, then joined: a subquery per count would read the views once per count (D-094).
|
|
98
|
+
hit_counts AS (
|
|
99
|
+
SELECT
|
|
100
|
+
period_kind,
|
|
101
|
+
from_month,
|
|
102
|
+
COUNT(*) AS hits,
|
|
103
|
+
SUM(window = 'five_hour') AS five_hour_hits,
|
|
104
|
+
SUM(window = 'seven_day') AS seven_day_hits,
|
|
105
|
+
SUM(window IS NULL) AS unknown_window_hits
|
|
106
|
+
FROM hits
|
|
107
|
+
GROUP BY period_kind, from_month
|
|
108
|
+
),
|
|
109
|
+
lock_counts AS (
|
|
110
|
+
SELECT period_kind, from_month, COUNT(*) AS lockouts, TOTAL(lockout_seconds) AS lockout_seconds
|
|
111
|
+
FROM obs_lockout_by_plan_events
|
|
112
|
+
GROUP BY period_kind, from_month
|
|
113
|
+
),
|
|
114
|
+
-- What a period's span is drawn from: request times and hit times, each with its local month.
|
|
115
|
+
dated AS MATERIALIZED (
|
|
116
|
+
SELECT timestamp_utc AS at_utc, strftime('%Y-%m', timestamp_utc, 'localtime') AS month
|
|
117
|
+
FROM requests_dedup WHERE timestamp_utc IS NOT NULL
|
|
118
|
+
UNION ALL
|
|
119
|
+
SELECT hit_at_utc, strftime('%Y-%m', hit_at_utc, 'localtime') FROM hits WHERE hit_at_utc IS NOT NULL
|
|
120
|
+
),
|
|
121
|
+
counted AS (
|
|
122
|
+
SELECT
|
|
123
|
+
p.period_kind,
|
|
124
|
+
p.from_month,
|
|
125
|
+
-- The last month the plan applies, for display: the month before until_month.
|
|
126
|
+
CASE WHEN p.until_month IS NOT NULL
|
|
127
|
+
THEN strftime('%Y-%m', p.until_month || '-01', '-1 month') END AS to_month,
|
|
128
|
+
p.until_month,
|
|
129
|
+
p.plan_name,
|
|
130
|
+
COALESCE(hc.hits, 0) AS hits,
|
|
131
|
+
COALESCE(hc.five_hour_hits, 0) AS five_hour_hits,
|
|
132
|
+
COALESCE(hc.seven_day_hits, 0) AS seven_day_hits,
|
|
133
|
+
COALESCE(hc.unknown_window_hits, 0) AS unknown_window_hits,
|
|
134
|
+
COALESCE(lc.lockouts, 0) AS lockouts,
|
|
135
|
+
-- TOTAL, as obs_lockout_time totals intervals: 0 over none.
|
|
136
|
+
COALESCE(lc.lockout_seconds, 0.0) AS lockout_seconds,
|
|
137
|
+
-- The span the period's data covers, from what is in it (principle 5); none for 'unknown'.
|
|
138
|
+
(SELECT MIN(d.at_utc) FROM dated d WHERE p.period_kind <> 'unknown'
|
|
139
|
+
AND (p.from_month IS NULL OR d.month >= p.from_month)
|
|
140
|
+
AND (p.until_month IS NULL OR d.month < p.until_month)) AS covers_from,
|
|
141
|
+
(SELECT MAX(d.at_utc) FROM dated d WHERE p.period_kind <> 'unknown'
|
|
142
|
+
AND (p.from_month IS NULL OR d.month >= p.from_month)
|
|
143
|
+
AND (p.until_month IS NULL OR d.month < p.until_month)) AS covers_to
|
|
144
|
+
FROM periods p
|
|
145
|
+
-- IS, not =: the 'none' and 'unknown' periods have no month, and NULL = NULL is never true.
|
|
146
|
+
LEFT JOIN hit_counts hc ON hc.period_kind = p.period_kind AND hc.from_month IS p.from_month
|
|
147
|
+
LEFT JOIN lock_counts lc ON lc.period_kind = p.period_kind AND lc.from_month IS p.from_month
|
|
148
|
+
)
|
|
149
|
+
SELECT *
|
|
150
|
+
FROM counted
|
|
151
|
+
-- A period appears when it holds a hit or any dated data; an entry for a month with no data, or
|
|
152
|
+
-- a plan entered for the future, has nothing to report.
|
|
153
|
+
WHERE hits > 0 OR covers_from IS NOT NULL;
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
-- 032_stored_views: requests_dedup and window_readings become tables that ingest fills from their views (D-096, step G6.1).
|
|
2
|
+
--
|
|
3
|
+
-- Every report statement recomputed both from the raw tables: window_readings pairs each of tens of thousands of
|
|
4
|
+
-- rate-limit readings with its API response through correlated lookups, and requests_dedup ranks every request line.
|
|
5
|
+
-- On real data the report and page ran about 20 statements built on them and took 45-50 s. Their inputs (requests,
|
|
6
|
+
-- parsed_lines, raw_lines, source_files, status_readings, rate_limit_windows) are written only by ingest, so ingest
|
|
7
|
+
-- stores their rows once (core/ingest/stored-views.ts) and everything downstream reads the stored rows. The
|
|
8
|
+
-- definitions are unchanged and keep running as views under *_computed, so the rows trace back exactly as before,
|
|
9
|
+
-- and a test fails if a stored table ever differs from its view.
|
|
10
|
+
--
|
|
11
|
+
-- Columns are ANY in a STRICT table: no type affinity, so every value is stored exactly as the view returned it.
|
|
12
|
+
|
|
13
|
+
DROP VIEW requests_dedup;
|
|
14
|
+
|
|
15
|
+
-- Observed: one row per distinct API response (winners) plus every unkeyed request line, as 019 computed it; ingest stores its rows in requests_dedup (D-096).
|
|
16
|
+
CREATE VIEW requests_dedup_computed AS
|
|
17
|
+
WITH ranked AS (
|
|
18
|
+
SELECT
|
|
19
|
+
r.raw_line_id,
|
|
20
|
+
r.dedup_key,
|
|
21
|
+
r.message_id,
|
|
22
|
+
r.request_id,
|
|
23
|
+
r.model,
|
|
24
|
+
r.input_tokens,
|
|
25
|
+
r.output_tokens,
|
|
26
|
+
r.cache_read_tokens,
|
|
27
|
+
r.cache_write_5m_tokens,
|
|
28
|
+
r.cache_write_1h_tokens,
|
|
29
|
+
r.cache_write_unsplit_tokens,
|
|
30
|
+
r.speed,
|
|
31
|
+
p.session_id,
|
|
32
|
+
p.timestamp_raw,
|
|
33
|
+
p.timestamp_utc,
|
|
34
|
+
p.cwd,
|
|
35
|
+
p.is_sidechain,
|
|
36
|
+
f.root,
|
|
37
|
+
f.relative_path,
|
|
38
|
+
l.first_run_id,
|
|
39
|
+
l.line_number,
|
|
40
|
+
ROW_NUMBER() OVER (
|
|
41
|
+
PARTITION BY r.dedup_key
|
|
42
|
+
ORDER BY r.output_tokens DESC, p.is_sidechain ASC,
|
|
43
|
+
-- Among lines tied on output, the earliest: when those numbers first existed
|
|
44
|
+
-- (D-065, the same rule D-045 applies to readings). NULLs last, so an unparsed
|
|
45
|
+
-- timestamp never wins.
|
|
46
|
+
(p.timestamp_utc IS NULL) ASC, p.timestamp_utc ASC,
|
|
47
|
+
f.root DESC, f.relative_path DESC, l.first_run_id DESC, l.line_number DESC
|
|
48
|
+
) AS candidate_rank
|
|
49
|
+
FROM requests r
|
|
50
|
+
JOIN parsed_lines p ON p.raw_line_id = r.raw_line_id
|
|
51
|
+
JOIN raw_lines l ON l.id = r.raw_line_id
|
|
52
|
+
JOIN source_files f ON f.id = l.source_file_id
|
|
53
|
+
)
|
|
54
|
+
SELECT
|
|
55
|
+
raw_line_id, dedup_key, message_id, request_id, model,
|
|
56
|
+
input_tokens, output_tokens, cache_read_tokens,
|
|
57
|
+
cache_write_5m_tokens, cache_write_1h_tokens, cache_write_unsplit_tokens, speed,
|
|
58
|
+
session_id, timestamp_raw, timestamp_utc, cwd, is_sidechain,
|
|
59
|
+
root, relative_path, first_run_id, line_number
|
|
60
|
+
FROM ranked
|
|
61
|
+
WHERE dedup_key IS NULL OR candidate_rank = 1;
|
|
62
|
+
|
|
63
|
+
CREATE TABLE requests_dedup (
|
|
64
|
+
raw_line_id ANY, -- derived: requests_dedup_computed.raw_line_id
|
|
65
|
+
dedup_key ANY, -- derived: requests_dedup_computed.dedup_key
|
|
66
|
+
message_id ANY, -- derived: requests_dedup_computed.message_id
|
|
67
|
+
request_id ANY, -- derived: requests_dedup_computed.request_id
|
|
68
|
+
model ANY, -- derived: requests_dedup_computed.model
|
|
69
|
+
input_tokens ANY, -- derived: requests_dedup_computed.input_tokens
|
|
70
|
+
output_tokens ANY, -- derived: requests_dedup_computed.output_tokens
|
|
71
|
+
cache_read_tokens ANY, -- derived: requests_dedup_computed.cache_read_tokens
|
|
72
|
+
cache_write_5m_tokens ANY, -- derived: requests_dedup_computed.cache_write_5m_tokens
|
|
73
|
+
cache_write_1h_tokens ANY, -- derived: requests_dedup_computed.cache_write_1h_tokens
|
|
74
|
+
cache_write_unsplit_tokens ANY, -- derived: requests_dedup_computed.cache_write_unsplit_tokens
|
|
75
|
+
speed ANY, -- derived: requests_dedup_computed.speed
|
|
76
|
+
session_id ANY, -- derived: requests_dedup_computed.session_id
|
|
77
|
+
timestamp_raw ANY, -- derived: requests_dedup_computed.timestamp_raw
|
|
78
|
+
timestamp_utc ANY, -- derived: requests_dedup_computed.timestamp_utc
|
|
79
|
+
cwd ANY, -- derived: requests_dedup_computed.cwd
|
|
80
|
+
is_sidechain ANY, -- derived: requests_dedup_computed.is_sidechain
|
|
81
|
+
root ANY, -- derived: requests_dedup_computed.root
|
|
82
|
+
relative_path ANY, -- derived: requests_dedup_computed.relative_path
|
|
83
|
+
first_run_id ANY, -- derived: requests_dedup_computed.first_run_id
|
|
84
|
+
line_number ANY -- derived: requests_dedup_computed.line_number
|
|
85
|
+
) STRICT;
|
|
86
|
+
|
|
87
|
+
CREATE INDEX requests_dedup_raw_line_id ON requests_dedup (raw_line_id);
|
|
88
|
+
CREATE INDEX requests_dedup_dedup_key ON requests_dedup (dedup_key);
|
|
89
|
+
CREATE INDEX requests_dedup_time ON requests_dedup (timestamp_utc);
|
|
90
|
+
CREATE INDEX requests_dedup_session_time ON requests_dedup (session_id, timestamp_utc);
|
|
91
|
+
CREATE INDEX requests_dedup_cwd ON requests_dedup (cwd);
|
|
92
|
+
|
|
93
|
+
INSERT INTO requests_dedup (raw_line_id, dedup_key, message_id, request_id, model, input_tokens, output_tokens, cache_read_tokens, cache_write_5m_tokens, cache_write_1h_tokens, cache_write_unsplit_tokens, speed, session_id, timestamp_raw, timestamp_utc, cwd, is_sidechain, root, relative_path, first_run_id, line_number)
|
|
94
|
+
SELECT raw_line_id, dedup_key, message_id, request_id, model, input_tokens, output_tokens, cache_read_tokens, cache_write_5m_tokens, cache_write_1h_tokens, cache_write_unsplit_tokens, speed, session_id, timestamp_raw, timestamp_utc, cwd, is_sidechain, root, relative_path, first_run_id, line_number FROM requests_dedup_computed;
|
|
95
|
+
|
|
96
|
+
DROP VIEW window_readings;
|
|
97
|
+
|
|
98
|
+
-- Observed: one row per observation of a plan window, as 028 computed it; ingest stores its rows in window_readings (D-096). As in 026 (D-044, D-045, D-086), with stale = 1 when a counted reading is more than 1 point below an earlier counted one in its window (D-089).
|
|
99
|
+
CREATE VIEW window_readings_computed AS
|
|
100
|
+
WITH readings AS (
|
|
101
|
+
SELECT
|
|
102
|
+
w.raw_line_id, w.window, w.resets_at, w.used_percentage, s.session_id, s.captured_at_s,
|
|
103
|
+
-- The response behind the latest request line in this session, inside this window, at or before
|
|
104
|
+
-- the capture (+1 s for whole-second captures). An unkeyed line is its own response.
|
|
105
|
+
(SELECT COALESCE(rq.dedup_key, 'line:' || rq.raw_line_id)
|
|
106
|
+
FROM parsed_lines p
|
|
107
|
+
JOIN requests rq ON rq.raw_line_id = p.raw_line_id
|
|
108
|
+
WHERE p.session_id = s.session_id AND p.class = 'request'
|
|
109
|
+
AND p.timestamp_utc > strftime('%Y-%m-%dT%H:%M:%fZ',
|
|
110
|
+
w.resets_at - CASE w.window WHEN 'five_hour' THEN 5 * 3600 ELSE 7 * 86400 END, 'unixepoch')
|
|
111
|
+
AND p.timestamp_utc < strftime('%Y-%m-%dT%H:%M:%fZ', s.captured_at_s + 1, 'unixepoch')
|
|
112
|
+
ORDER BY p.timestamp_utc DESC, p.raw_line_id DESC
|
|
113
|
+
LIMIT 1) AS response_key
|
|
114
|
+
FROM rate_limit_windows w
|
|
115
|
+
JOIN status_readings s ON s.raw_line_id = w.raw_line_id
|
|
116
|
+
WHERE s.status = 'ok' AND w.validity = 'valid' AND w.window IN ('five_hour', 'seven_day')
|
|
117
|
+
),
|
|
118
|
+
dated AS (
|
|
119
|
+
SELECT
|
|
120
|
+
r.*,
|
|
121
|
+
-- The response's first line: when its numbers first existed.
|
|
122
|
+
(SELECT MIN(p.timestamp_utc)
|
|
123
|
+
FROM requests rq
|
|
124
|
+
JOIN parsed_lines p ON p.raw_line_id = rq.raw_line_id
|
|
125
|
+
WHERE (r.response_key LIKE 'line:%' AND rq.raw_line_id = CAST(substr(r.response_key, 6) AS INTEGER))
|
|
126
|
+
OR rq.dedup_key = r.response_key) AS response_first_utc,
|
|
127
|
+
-- The previous capture of the same response, in capture order.
|
|
128
|
+
LAG(r.used_percentage) OVER captures AS previous_used_percentage
|
|
129
|
+
FROM readings r
|
|
130
|
+
-- A capture-time fallback is its own group: NULL response keys must not merge (D-044).
|
|
131
|
+
WINDOW captures AS (
|
|
132
|
+
PARTITION BY r.window, r.resets_at, r.session_id, r.response_key,
|
|
133
|
+
CASE WHEN r.response_key IS NULL THEN r.raw_line_id END
|
|
134
|
+
ORDER BY r.captured_at_s, r.raw_line_id
|
|
135
|
+
)
|
|
136
|
+
),
|
|
137
|
+
runs AS (
|
|
138
|
+
SELECT
|
|
139
|
+
d.*,
|
|
140
|
+
-- A new run starts at the first capture and at every change of more than 0.01 points: payloads
|
|
141
|
+
-- carry computed values such as 7.000000000000001, which are the same number (D-044, D-086).
|
|
142
|
+
SUM(CASE WHEN d.previous_used_percentage IS NULL
|
|
143
|
+
OR abs(d.used_percentage - d.previous_used_percentage) > 0.01 THEN 1 ELSE 0 END)
|
|
144
|
+
OVER (
|
|
145
|
+
PARTITION BY d.window, d.resets_at, d.session_id, d.response_key,
|
|
146
|
+
CASE WHEN d.response_key IS NULL THEN d.raw_line_id END
|
|
147
|
+
ORDER BY d.captured_at_s, d.raw_line_id
|
|
148
|
+
ROWS UNBOUNDED PRECEDING
|
|
149
|
+
) AS run
|
|
150
|
+
FROM dated d
|
|
151
|
+
),
|
|
152
|
+
observed AS (
|
|
153
|
+
SELECT
|
|
154
|
+
u.*,
|
|
155
|
+
-- A later run's value didn't come from the response, so it's dated when it was first seen.
|
|
156
|
+
CASE
|
|
157
|
+
WHEN u.response_first_utc IS NULL THEN strftime('%Y-%m-%dT%H:%M:%fZ', u.captured_at_s, 'unixepoch')
|
|
158
|
+
WHEN u.run = 1 THEN u.response_first_utc
|
|
159
|
+
ELSE strftime('%Y-%m-%dT%H:%M:%fZ', MIN(u.captured_at_s) OVER run_captures, 'unixepoch')
|
|
160
|
+
END AS observed_at_utc,
|
|
161
|
+
ROW_NUMBER() OVER run_captures_latest AS copy_rank
|
|
162
|
+
FROM runs u
|
|
163
|
+
WINDOW
|
|
164
|
+
run_captures AS (
|
|
165
|
+
PARTITION BY u.window, u.resets_at, u.session_id, u.response_key,
|
|
166
|
+
CASE WHEN u.response_key IS NULL THEN u.raw_line_id END, u.run
|
|
167
|
+
),
|
|
168
|
+
run_captures_latest AS (
|
|
169
|
+
PARTITION BY u.window, u.resets_at, u.session_id, u.response_key,
|
|
170
|
+
CASE WHEN u.response_key IS NULL THEN u.raw_line_id END, u.run
|
|
171
|
+
ORDER BY u.captured_at_s DESC, u.raw_line_id DESC
|
|
172
|
+
)
|
|
173
|
+
),
|
|
174
|
+
counted AS (
|
|
175
|
+
SELECT
|
|
176
|
+
raw_line_id,
|
|
177
|
+
window,
|
|
178
|
+
resets_at,
|
|
179
|
+
strftime('%Y-%m-%dT%H:%M:%fZ', resets_at, 'unixepoch') AS reset_at_utc,
|
|
180
|
+
used_percentage,
|
|
181
|
+
session_id,
|
|
182
|
+
captured_at_s,
|
|
183
|
+
strftime('%Y-%m-%dT%H:%M:%fZ', captured_at_s, 'unixepoch') AS captured_at_utc,
|
|
184
|
+
observed_at_utc,
|
|
185
|
+
unixepoch(observed_at_utc, 'subsec') AS observed_at_s,
|
|
186
|
+
CASE
|
|
187
|
+
WHEN response_key IS NULL THEN 'capture'
|
|
188
|
+
WHEN run = 1 THEN 'last_request'
|
|
189
|
+
ELSE 'changed_capture'
|
|
190
|
+
END AS observed_via,
|
|
191
|
+
unixepoch(observed_at_utc, 'subsec') > resets_at AS after_reset
|
|
192
|
+
FROM observed
|
|
193
|
+
WHERE copy_rank = 1
|
|
194
|
+
)
|
|
195
|
+
SELECT
|
|
196
|
+
c.*,
|
|
197
|
+
-- Usage within one window only rises, so a counted reading more than 1 point below an earlier one
|
|
198
|
+
-- in the same window, from any session, carries a number from before that earlier reading (D-089).
|
|
199
|
+
CASE
|
|
200
|
+
WHEN c.after_reset = 0 AND c.used_percentage < MAX(c.used_percentage) OVER (
|
|
201
|
+
PARTITION BY c.window, c.resets_at ORDER BY c.observed_at_s, c.raw_line_id
|
|
202
|
+
ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING
|
|
203
|
+
) - 1 THEN 1
|
|
204
|
+
ELSE 0
|
|
205
|
+
END AS stale
|
|
206
|
+
FROM counted c;
|
|
207
|
+
|
|
208
|
+
CREATE TABLE window_readings (
|
|
209
|
+
raw_line_id ANY, -- derived: window_readings_computed.raw_line_id
|
|
210
|
+
window ANY, -- derived: window_readings_computed.window
|
|
211
|
+
resets_at ANY, -- derived: window_readings_computed.resets_at
|
|
212
|
+
reset_at_utc ANY, -- derived: window_readings_computed.reset_at_utc
|
|
213
|
+
used_percentage ANY, -- derived: window_readings_computed.used_percentage
|
|
214
|
+
session_id ANY, -- derived: window_readings_computed.session_id
|
|
215
|
+
captured_at_s ANY, -- derived: window_readings_computed.captured_at_s
|
|
216
|
+
captured_at_utc ANY, -- derived: window_readings_computed.captured_at_utc
|
|
217
|
+
observed_at_utc ANY, -- derived: window_readings_computed.observed_at_utc
|
|
218
|
+
observed_at_s ANY, -- derived: window_readings_computed.observed_at_s
|
|
219
|
+
observed_via ANY, -- derived: window_readings_computed.observed_via
|
|
220
|
+
after_reset ANY, -- derived: window_readings_computed.after_reset
|
|
221
|
+
stale ANY -- derived: window_readings_computed.stale
|
|
222
|
+
) STRICT;
|
|
223
|
+
|
|
224
|
+
CREATE INDEX window_readings_raw_line ON window_readings (raw_line_id);
|
|
225
|
+
CREATE INDEX window_readings_instance ON window_readings (window, reset_at_utc, observed_at_s);
|
|
226
|
+
CREATE INDEX window_readings_session_time ON window_readings (session_id, observed_at_s);
|
|
227
|
+
CREATE INDEX window_readings_time ON window_readings (observed_at_utc);
|
|
228
|
+
|
|
229
|
+
INSERT INTO window_readings (raw_line_id, window, resets_at, reset_at_utc, used_percentage, session_id, captured_at_s, captured_at_utc, observed_at_utc, observed_at_s, observed_via, after_reset, stale)
|
|
230
|
+
SELECT raw_line_id, window, resets_at, reset_at_utc, used_percentage, session_id, captured_at_s, captured_at_utc, observed_at_utc, observed_at_s, observed_via, after_reset, stale FROM window_readings_computed;
|
|
231
|
+
|
|
232
|
+
-- Statistics for the query planner, so the first report after this migration is fast too (D-096).
|
|
233
|
+
ANALYZE;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* @file Command-line entry point. Holds no logic, so it is excluded from coverage; everything it
|
|
4
|
+
* runs is tested through `checkNodeVersion()` in node-version.ts, `runCli()` in program.ts, and
|
|
5
|
+
* the CLI integration tests.
|
|
6
|
+
*/
|
|
7
|
+
import { homedir } from "node:os";
|
|
8
|
+
import { dirname } from "node:path";
|
|
9
|
+
import { fileURLToPath } from "node:url";
|
|
10
|
+
import { checkNodeVersion } from "../core/util/node-version.js";
|
|
11
|
+
// Checked before the program is imported: on an older Node.js, the built-in `node:sqlite` is missing
|
|
12
|
+
// (22.11 throws on import without a flag) or experimental (22.23 warns on every run), and Nilometer
|
|
13
|
+
// isn't tested there (D-066, D-093). The program is loaded with a dynamic import below so nothing
|
|
14
|
+
// that loads `node:sqlite` runs before this check.
|
|
15
|
+
const nodeCheck = checkNodeVersion(process.versions.node);
|
|
16
|
+
if (nodeCheck.message !== undefined) {
|
|
17
|
+
console.error(nodeCheck.message);
|
|
18
|
+
}
|
|
19
|
+
if (nodeCheck.status === "too_old") {
|
|
20
|
+
process.exitCode = 1;
|
|
21
|
+
}
|
|
22
|
+
else {
|
|
23
|
+
const { findPackageRoot } = await import("../core/install/locations.js");
|
|
24
|
+
const { runCli } = await import("./program.js");
|
|
25
|
+
process.exitCode = runCli(process.argv.slice(2), {
|
|
26
|
+
home: homedir(),
|
|
27
|
+
env: process.env,
|
|
28
|
+
packageRoot: findPackageRoot(dirname(fileURLToPath(import.meta.url))),
|
|
29
|
+
now: () => new Date(),
|
|
30
|
+
print: (line) => {
|
|
31
|
+
console.log(line);
|
|
32
|
+
},
|
|
33
|
+
printError: (line) => {
|
|
34
|
+
console.error(line);
|
|
35
|
+
},
|
|
36
|
+
timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone,
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
//# sourceMappingURL=main.cli.js.map
|