@columenlabs/lumen-ops 0.0.0-stage → 0.1.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/NOTICE +12 -0
- package/README.md +97 -2
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/points.d.ts +8 -0
- package/dist/points.js +1 -0
- package/dist/queries.d.ts +75 -0
- package/dist/queries.js +412 -0
- package/package.json +41 -4
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Columen Labs
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/NOTICE
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
Lumen
|
|
2
|
+
Copyright (c) 2026 Columen Labs. Licensed under the MIT License (see LICENSE).
|
|
3
|
+
|
|
4
|
+
Lumen draws price and market charts with TradingView Lightweight Charts™
|
|
5
|
+
(https://github.com/tradingview/lightweight-charts), Copyright (c) 2025
|
|
6
|
+
TradingView, Inc., licensed under the Apache License 2.0. Lumen turns off the
|
|
7
|
+
chart's own logo, so per that library's NOTICE, an app that shows these charts
|
|
8
|
+
should link to https://www.tradingview.com/ somewhere a user can find it (an
|
|
9
|
+
About page or footer is enough).
|
|
10
|
+
|
|
11
|
+
The components under packages/ui/src/ui are based on shadcn/ui
|
|
12
|
+
(https://ui.shadcn.com), MIT License, Copyright (c) 2023 shadcn.
|
package/README.md
CHANGED
|
@@ -1,3 +1,98 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @columenlabs/lumen-ops
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Lumen's ops meters for Cloudflare Workers: every data query your app runs becomes one Analytics Engine point, so you can find slow and heavy queries and fix them. The meter wraps your bindings once, so new code is metered without opting in. It writes only each query's shape, never a value.
|
|
4
|
+
|
|
5
|
+
- Docs: https://lumen.columenlabs.com/docs/
|
|
6
|
+
- For agents: https://lumen.columenlabs.com/llms.txt
|
|
7
|
+
- Source: https://github.com/columenlabs/lumen (`packages/ops`). MIT.
|
|
8
|
+
|
|
9
|
+
## Install
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
bun add @columenlabs/lumen-ops
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
Pin it to the same exact version as your other Lumen packages. It uses `AsyncLocalStorage`, so the Worker needs the `nodejs_compat` flag.
|
|
16
|
+
|
|
17
|
+
## Set up
|
|
18
|
+
|
|
19
|
+
Bind an Analytics Engine dataset in `wrangler.jsonc`:
|
|
20
|
+
|
|
21
|
+
```jsonc
|
|
22
|
+
"analytics_engine_datasets": [{ "binding": "QUERIES", "dataset": "myapp_queries" }]
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Wrap the bindings once at start, and hand the wrapped ones to your app:
|
|
26
|
+
|
|
27
|
+
```ts
|
|
28
|
+
import { foldRoute, meteredClickHouse, meteredD1, setQueryCallerOf, setQuerySink } from '@columenlabs/lumen-ops';
|
|
29
|
+
|
|
30
|
+
setQuerySink(env.QUERIES);
|
|
31
|
+
// Name the request in scope: the app (or part of it), the route, and the tool when an MCP call runs.
|
|
32
|
+
setQueryCallerOf(() => {
|
|
33
|
+
const req = currentRequest(); // your own request scope
|
|
34
|
+
return req && { app: 'myapp', kind: 'fetch', route: foldRoute(new URL(req.url).pathname) };
|
|
35
|
+
});
|
|
36
|
+
const db = meteredD1(env.DB);
|
|
37
|
+
const clickhouse = meteredClickHouse(env.CLICKHOUSE); // any ClickHouse HTTP service binding
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
In a Durable Object, meter its SQLite:
|
|
41
|
+
|
|
42
|
+
```ts
|
|
43
|
+
import { meteredSql } from '@columenlabs/lumen-ops';
|
|
44
|
+
|
|
45
|
+
const sql = meteredSql(state.storage.sql, { app: 'myapp', kind: 'do', route: 'Person' });
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
A cron run, queue batch or alarm names itself, so its queries are charged to it:
|
|
49
|
+
|
|
50
|
+
```ts
|
|
51
|
+
import { withQueryCaller } from '@columenlabs/lumen-ops';
|
|
52
|
+
|
|
53
|
+
await withQueryCaller({ app: 'myapp', kind: 'cron', route: 'nightly-rollup' }, () => rollup(env));
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
Another engine (R2 SQL, an HTTP database) calls `meterQuery({ engine, sql, method, ok, ms, ... })` from its one client.
|
|
57
|
+
|
|
58
|
+
## What a point holds
|
|
59
|
+
|
|
60
|
+
One point per statement (a D1 batch writes one per statement in it). The column order is the contract for reading the dataset back:
|
|
61
|
+
|
|
62
|
+
| Column | Holds |
|
|
63
|
+
|---|---|
|
|
64
|
+
| `index1` | Engine: `d1`, `clickhouse`, `do_sqlite`, `r2_sql` or your own |
|
|
65
|
+
| `blob1` … `blob5` | Engine, app, kind (`fetch`, `cron`, `queue`, `alarm`, `do`), route (ids folded to `:id`), tool |
|
|
66
|
+
| `blob6` … `blob8` | Op (`select`, `insert`, `update`, `delete`, `ddl`, `pragma`, `other`), fingerprint, the query's shape (up to 1 KB) |
|
|
67
|
+
| `blob9`, `blob10` | `ok` or `error`, method (`all`, `first`, `run`, `raw`, `batch`, `query`, `insert`, `exec`) |
|
|
68
|
+
| `double1` … `double8` | Wall ms, the engine's own ms, rows read, bytes read, rows returned, rows written, response bytes, HTTP status |
|
|
69
|
+
|
|
70
|
+
The shape has every string and number literal as `?`, `IN` lists folded to `(?...)`, comments and ClickHouse's `FORMAT` dropped, and an insert's rows cut off. The fingerprint is a hash of the shape, one per kind of query.
|
|
71
|
+
|
|
72
|
+
Read it back with the Analytics Engine SQL API, for example the slowest shapes of the last day:
|
|
73
|
+
|
|
74
|
+
```sql
|
|
75
|
+
SELECT blob7 AS fingerprint, any(blob8) AS shape, sum(_sample_interval) AS calls,
|
|
76
|
+
quantileWeighted(0.95)(double1, _sample_interval) AS p95_ms
|
|
77
|
+
FROM myapp_queries
|
|
78
|
+
WHERE timestamp > NOW() - INTERVAL '1' DAY
|
|
79
|
+
GROUP BY fingerprint ORDER BY p95_ms DESC LIMIT 20
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
## Exports
|
|
83
|
+
|
|
84
|
+
| Export | What it does |
|
|
85
|
+
|---|---|
|
|
86
|
+
| `setQuerySink(binding)` | The dataset every point goes to; nothing is written until it is set |
|
|
87
|
+
| `setQueryCallerOf(fn)` | Names the request in scope when no job is |
|
|
88
|
+
| `withQueryCaller(caller, fn)` | Charges every query inside `fn` to `caller` |
|
|
89
|
+
| `meteredD1(db)` | D1, metered: `prepare` and `batch`, with D1's own rows read and written and duration |
|
|
90
|
+
| `meteredClickHouse(binding)` | A ClickHouse HTTP service binding, metered when the answer has been read: wall time to the last byte, ClickHouse's own statistics |
|
|
91
|
+
| `meteredSql(sql, caller)` | A Durable Object's `storage.sql`, metered |
|
|
92
|
+
| `meterQuery(point)` | One point by hand, for any other engine |
|
|
93
|
+
| `queryShape(sql)` | The shape, fingerprint and op of a query |
|
|
94
|
+
| `foldRoute(path)` | A path with id-like segments as `:id` |
|
|
95
|
+
|
|
96
|
+
## Versions
|
|
97
|
+
|
|
98
|
+
0.x: any minor may break things until 1.0. Every merge to Lumen's `main` publishes the next patch of every Lumen package; see the [releases](https://github.com/columenlabs/lumen/releases).
|
package/dist/index.d.ts
ADDED
package/dist/index.js
ADDED
package/dist/points.d.ts
ADDED
package/dist/points.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import type { PointSink } from './points.js';
|
|
2
|
+
export type QueryEngine = 'clickhouse' | 'd1' | 'do_sqlite' | 'r2_sql' | (string & {});
|
|
3
|
+
export interface QueryCaller {
|
|
4
|
+
/** The app, or the part of it, the query ran for. */
|
|
5
|
+
app: string;
|
|
6
|
+
/** fetch, cron, queue, alarm, ws or do. */
|
|
7
|
+
kind: string;
|
|
8
|
+
/** A route (folded path), or the job's name. */
|
|
9
|
+
route: string;
|
|
10
|
+
tool?: string;
|
|
11
|
+
}
|
|
12
|
+
export interface QueryPoint {
|
|
13
|
+
engine: QueryEngine;
|
|
14
|
+
/** The SQL as sent; only its normalized shape is written. */
|
|
15
|
+
sql: string;
|
|
16
|
+
/** How it was called: query, insert (ClickHouse); all, first, run, raw, batch (D1); exec. */
|
|
17
|
+
method: string;
|
|
18
|
+
ok: boolean;
|
|
19
|
+
/** Wall time in the Worker, ms (request to last byte). */
|
|
20
|
+
ms: number;
|
|
21
|
+
/** The engine's own time, ms, where it says. */
|
|
22
|
+
engineMs?: number;
|
|
23
|
+
rowsRead?: number;
|
|
24
|
+
bytesRead?: number;
|
|
25
|
+
/** Rows the query returned. */
|
|
26
|
+
rowsOut?: number;
|
|
27
|
+
rowsWritten?: number;
|
|
28
|
+
/** Response bytes (ClickHouse). */
|
|
29
|
+
bytesOut?: number;
|
|
30
|
+
/** HTTP status (ClickHouse). */
|
|
31
|
+
status?: number;
|
|
32
|
+
/** Defaults to the request or job in scope. */
|
|
33
|
+
caller?: QueryCaller;
|
|
34
|
+
}
|
|
35
|
+
/** The entry point sets the dataset once per isolate. */
|
|
36
|
+
export declare function setQuerySink(s: PointSink | undefined): void;
|
|
37
|
+
/** How to name the request in scope when no job is (the app's request telemetry). */
|
|
38
|
+
export declare function setQueryCallerOf(f: (() => QueryCaller | undefined) | undefined): void;
|
|
39
|
+
/** Runs `fn` with `caller` as the caller of every query inside (cron runs, queue batches, alarms). */
|
|
40
|
+
export declare function withQueryCaller<T>(caller: QueryCaller, fn: () => T): T;
|
|
41
|
+
/** Id-like path segments become `:id`, so a route never carries an id. */
|
|
42
|
+
export declare const foldRoute: (path: string) => string;
|
|
43
|
+
export interface QueryShape {
|
|
44
|
+
/** The SQL with every literal as `?`, lists folded, comments and the output format dropped. */
|
|
45
|
+
text: string;
|
|
46
|
+
/** A hash of `text`: one per query shape. */
|
|
47
|
+
fingerprint: string;
|
|
48
|
+
/** select, insert, update, delete, ddl, pragma or other. */
|
|
49
|
+
op: string;
|
|
50
|
+
}
|
|
51
|
+
/** The query's shape: no literal survives, so no value or person's data is ever written. */
|
|
52
|
+
export declare function queryShape(sql: string): QueryShape;
|
|
53
|
+
/**
|
|
54
|
+
* Column order is the contract with whoever reads the dataset back: index1 engine; blobs
|
|
55
|
+
* engine, app, kind, route, tool, op, fingerprint, sql shape, outcome, method; doubles wall ms,
|
|
56
|
+
* engine ms, rows read, bytes read, rows out, rows written, bytes out, status.
|
|
57
|
+
*/
|
|
58
|
+
export declare function meterQuery(p: QueryPoint): void;
|
|
59
|
+
/** The D1 binding, metered: one point per statement, a batch's each with its own meta. */
|
|
60
|
+
export declare function meteredD1<T>(db: T | undefined): T | undefined;
|
|
61
|
+
type Fetch = (input: Request | string | URL, init?: RequestInit) => Promise<Response>;
|
|
62
|
+
/**
|
|
63
|
+
* The ClickHouse VPC binding, metered. The point is written when the caller has read the whole
|
|
64
|
+
* answer: wall time to the last byte, and the `statistics` ClickHouse puts at the end of a JSON
|
|
65
|
+
* answer (rows and bytes read, its own time), else the summary header.
|
|
66
|
+
*/
|
|
67
|
+
export declare function meteredClickHouse<T extends {
|
|
68
|
+
fetch: Fetch;
|
|
69
|
+
}>(binding: T | undefined): T | undefined;
|
|
70
|
+
/**
|
|
71
|
+
* A Durable Object's `storage.sql`, metered for `caller`. Each statement is read to the end at
|
|
72
|
+
* `exec`, so its rows read and written are known; callers read rows only through `toArray`.
|
|
73
|
+
*/
|
|
74
|
+
export declare function meteredSql<T>(sql: T, caller: QueryCaller): T;
|
|
75
|
+
export {};
|
package/dist/queries.js
ADDED
|
@@ -0,0 +1,412 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The query meter: every data query an app runs is one Analytics Engine point, with the engine it
|
|
3
|
+
* ran on, the query's shape and what it cost, so slow and heavy queries can be found and fixed.
|
|
4
|
+
*
|
|
5
|
+
* It sits on the bindings, not the call sites: wrap D1 with `meteredD1`, a ClickHouse HTTP service
|
|
6
|
+
* with `meteredClickHouse` and a Durable Object's SQLite with `meteredSql` once at start, and every
|
|
7
|
+
* query through them is metered without opting in. Another engine calls `meterQuery` from its one
|
|
8
|
+
* client.
|
|
9
|
+
*
|
|
10
|
+
* A point holds the query's shape only: string and number literals become `?` and `IN` lists fold,
|
|
11
|
+
* so no value, id or person's data is ever written. The caller is the app, route and tool of the
|
|
12
|
+
* request (or the cron job, queue batch or alarm) it ran in: `withQueryCaller` names a job, and
|
|
13
|
+
* `setQueryCallerOf` tells the meter how to name the request in scope.
|
|
14
|
+
*/
|
|
15
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
16
|
+
let sink;
|
|
17
|
+
let callerOf;
|
|
18
|
+
const scope = new AsyncLocalStorage();
|
|
19
|
+
/** The entry point sets the dataset once per isolate. */
|
|
20
|
+
export function setQuerySink(s) {
|
|
21
|
+
sink = s;
|
|
22
|
+
}
|
|
23
|
+
/** How to name the request in scope when no job is (the app's request telemetry). */
|
|
24
|
+
export function setQueryCallerOf(f) {
|
|
25
|
+
callerOf = f;
|
|
26
|
+
}
|
|
27
|
+
/** Runs `fn` with `caller` as the caller of every query inside (cron runs, queue batches, alarms). */
|
|
28
|
+
export function withQueryCaller(caller, fn) {
|
|
29
|
+
return scope.run(caller, fn);
|
|
30
|
+
}
|
|
31
|
+
function currentCaller() {
|
|
32
|
+
const job = scope.getStore();
|
|
33
|
+
if (job)
|
|
34
|
+
return job;
|
|
35
|
+
try {
|
|
36
|
+
const c = callerOf?.();
|
|
37
|
+
if (c)
|
|
38
|
+
return c;
|
|
39
|
+
}
|
|
40
|
+
catch { }
|
|
41
|
+
return { app: 'other', kind: 'other', route: '' };
|
|
42
|
+
}
|
|
43
|
+
/** Id-like path segments become `:id`, so a route never carries an id. */
|
|
44
|
+
export const foldRoute = (path) => path
|
|
45
|
+
.replace(/^\/api\/v1(?=\/|$)/, '/api')
|
|
46
|
+
.split('/')
|
|
47
|
+
.map((seg) => /^[a-z]{1,3}_[A-Za-z0-9]+$/.test(seg) ||
|
|
48
|
+
seg.length >= 20 ||
|
|
49
|
+
(seg.length >= 8 && /\d/.test(seg))
|
|
50
|
+
? ':id'
|
|
51
|
+
: seg)
|
|
52
|
+
.join('/');
|
|
53
|
+
const SQL_CAP = 1024;
|
|
54
|
+
const shapes = new Map();
|
|
55
|
+
const fnv = (s, seed) => {
|
|
56
|
+
let h = seed >>> 0;
|
|
57
|
+
for (let i = 0; i < s.length; i++)
|
|
58
|
+
h = Math.imul(h ^ s.charCodeAt(i), 16777619) >>> 0;
|
|
59
|
+
return h.toString(16).padStart(8, '0');
|
|
60
|
+
};
|
|
61
|
+
/** The query's shape: no literal survives, so no value or person's data is ever written. */
|
|
62
|
+
export function queryShape(sql) {
|
|
63
|
+
const known = shapes.get(sql);
|
|
64
|
+
if (known)
|
|
65
|
+
return known;
|
|
66
|
+
let s = sql;
|
|
67
|
+
// An insert's rows ride after its FORMAT line; only the statement is kept.
|
|
68
|
+
const insert = /^\s*INSERT\b/i.test(s);
|
|
69
|
+
if (insert)
|
|
70
|
+
s = s.split('\n', 1)[0] ?? '';
|
|
71
|
+
s = s
|
|
72
|
+
.replace(/--[^\n]*/g, ' ')
|
|
73
|
+
.replace(/\/\*[\s\S]*?\*\//g, ' ')
|
|
74
|
+
.replace(/'(?:[^'\\]|\\.|'')*'/g, '?')
|
|
75
|
+
.replace(/(?<![\w$.])\d+(?:\.\d+)?(?:e[+-]?\d+)?\b/gi, '?')
|
|
76
|
+
.replace(/\s+/g, ' ')
|
|
77
|
+
.trim();
|
|
78
|
+
if (!insert)
|
|
79
|
+
s = s.replace(/\s+FORMAT\s+\w+\s*$/i, '');
|
|
80
|
+
s = s
|
|
81
|
+
.replace(/\(\s*\?(?:\s*,\s*\?)+\s*\)/g, '(?...)')
|
|
82
|
+
.replace(/\[\s*\?(?:\s*,\s*\?)+\s*\]/g, '[?...]')
|
|
83
|
+
.replace(/\(\?\.\.\.\)(?:\s*,\s*\(\?\.\.\.\))+/g, '(?...),...');
|
|
84
|
+
const head = (s.match(/^\(?\s*(\w+)/)?.[1] ?? '').toUpperCase();
|
|
85
|
+
const op = head === 'SELECT' || head === 'WITH'
|
|
86
|
+
? 'select'
|
|
87
|
+
: head === 'INSERT' || head === 'REPLACE'
|
|
88
|
+
? 'insert'
|
|
89
|
+
: head === 'UPDATE'
|
|
90
|
+
? 'update'
|
|
91
|
+
: head === 'DELETE'
|
|
92
|
+
? 'delete'
|
|
93
|
+
: head === 'CREATE' || head === 'ALTER' || head === 'DROP'
|
|
94
|
+
? 'ddl'
|
|
95
|
+
: head === 'PRAGMA'
|
|
96
|
+
? 'pragma'
|
|
97
|
+
: 'other';
|
|
98
|
+
const shape = {
|
|
99
|
+
text: s.slice(0, SQL_CAP),
|
|
100
|
+
fingerprint: fnv(s, 2166136261) + fnv(s, 0x9e3779b9),
|
|
101
|
+
op,
|
|
102
|
+
};
|
|
103
|
+
if (shapes.size >= 1000)
|
|
104
|
+
shapes.clear();
|
|
105
|
+
shapes.set(sql, shape);
|
|
106
|
+
return shape;
|
|
107
|
+
}
|
|
108
|
+
const cap = (s, n = 96) => (s.length > n ? s.slice(0, n) : s);
|
|
109
|
+
const num = (v) => (typeof v === 'number' && Number.isFinite(v) ? v : Number(v) || 0);
|
|
110
|
+
/**
|
|
111
|
+
* Column order is the contract with whoever reads the dataset back: index1 engine; blobs
|
|
112
|
+
* engine, app, kind, route, tool, op, fingerprint, sql shape, outcome, method; doubles wall ms,
|
|
113
|
+
* engine ms, rows read, bytes read, rows out, rows written, bytes out, status.
|
|
114
|
+
*/
|
|
115
|
+
export function meterQuery(p) {
|
|
116
|
+
if (!sink)
|
|
117
|
+
return;
|
|
118
|
+
try {
|
|
119
|
+
const shape = queryShape(p.sql);
|
|
120
|
+
const c = p.caller ?? currentCaller();
|
|
121
|
+
sink.writeDataPoint({
|
|
122
|
+
indexes: [p.engine],
|
|
123
|
+
blobs: [
|
|
124
|
+
p.engine,
|
|
125
|
+
cap(c.app),
|
|
126
|
+
cap(c.kind),
|
|
127
|
+
cap(c.route),
|
|
128
|
+
cap(c.tool ?? ''),
|
|
129
|
+
shape.op,
|
|
130
|
+
shape.fingerprint,
|
|
131
|
+
shape.text,
|
|
132
|
+
p.ok ? 'ok' : 'error',
|
|
133
|
+
p.method,
|
|
134
|
+
],
|
|
135
|
+
doubles: [
|
|
136
|
+
p.ms,
|
|
137
|
+
num(p.engineMs),
|
|
138
|
+
num(p.rowsRead),
|
|
139
|
+
num(p.bytesRead),
|
|
140
|
+
num(p.rowsOut),
|
|
141
|
+
num(p.rowsWritten),
|
|
142
|
+
num(p.bytesOut),
|
|
143
|
+
num(p.status),
|
|
144
|
+
],
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
catch (err) {
|
|
148
|
+
console.error('meterQuery', err);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
const d1Point = (sql, method, ms, r) => meterQuery({
|
|
152
|
+
engine: 'd1',
|
|
153
|
+
sql,
|
|
154
|
+
method,
|
|
155
|
+
ok: Boolean(r),
|
|
156
|
+
ms,
|
|
157
|
+
engineMs: r?.meta?.duration,
|
|
158
|
+
rowsRead: r?.meta?.rows_read,
|
|
159
|
+
rowsWritten: r?.meta?.rows_written,
|
|
160
|
+
rowsOut: r?.results?.length,
|
|
161
|
+
});
|
|
162
|
+
class MeteredStatement {
|
|
163
|
+
native;
|
|
164
|
+
sql;
|
|
165
|
+
constructor(native, sql) {
|
|
166
|
+
this.native = native;
|
|
167
|
+
this.sql = sql;
|
|
168
|
+
}
|
|
169
|
+
bind(...values) {
|
|
170
|
+
return new MeteredStatement(this.native.bind(...values), this.sql);
|
|
171
|
+
}
|
|
172
|
+
async timed(method, run) {
|
|
173
|
+
const t0 = performance.now();
|
|
174
|
+
try {
|
|
175
|
+
const r = await run();
|
|
176
|
+
d1Point(this.sql, method, performance.now() - t0, r);
|
|
177
|
+
return r;
|
|
178
|
+
}
|
|
179
|
+
catch (err) {
|
|
180
|
+
d1Point(this.sql, method, performance.now() - t0);
|
|
181
|
+
throw err;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
all() {
|
|
185
|
+
return this.timed('all', () => this.native.all());
|
|
186
|
+
}
|
|
187
|
+
run() {
|
|
188
|
+
return this.timed('run', () => this.native.run());
|
|
189
|
+
}
|
|
190
|
+
/** D1's own `first` is `all` and its first row; through `all` it carries rows read. */
|
|
191
|
+
async first(column) {
|
|
192
|
+
const r = await this.timed('first', () => this.native.all());
|
|
193
|
+
const row = (r.results?.[0] ?? null);
|
|
194
|
+
if (!row || column === undefined)
|
|
195
|
+
return row;
|
|
196
|
+
if (!(column in row))
|
|
197
|
+
throw new Error(`D1_COLUMN_NOTFOUND: Column not found (${column})`);
|
|
198
|
+
return row[column];
|
|
199
|
+
}
|
|
200
|
+
async raw(opts) {
|
|
201
|
+
const t0 = performance.now();
|
|
202
|
+
let ok = false;
|
|
203
|
+
try {
|
|
204
|
+
const rows = await this.native.raw(opts);
|
|
205
|
+
ok = true;
|
|
206
|
+
return rows;
|
|
207
|
+
}
|
|
208
|
+
finally {
|
|
209
|
+
meterQuery({ engine: 'd1', sql: this.sql, method: 'raw', ok, ms: performance.now() - t0 });
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
/** The D1 binding, metered: one point per statement, a batch's each with its own meta. */
|
|
214
|
+
export function meteredD1(db) {
|
|
215
|
+
if (!db)
|
|
216
|
+
return db;
|
|
217
|
+
const native = db;
|
|
218
|
+
return new Proxy(native, {
|
|
219
|
+
get(target, prop, receiver) {
|
|
220
|
+
if (prop === 'prepare')
|
|
221
|
+
return (sql) => new MeteredStatement(target.prepare(sql), sql);
|
|
222
|
+
if (prop === 'batch')
|
|
223
|
+
return async (stmts) => {
|
|
224
|
+
const t0 = performance.now();
|
|
225
|
+
const unwrap = stmts.map((s) => (s instanceof MeteredStatement ? s.native : s));
|
|
226
|
+
const sqls = stmts.map((s) => (s instanceof MeteredStatement ? s.sql : ''));
|
|
227
|
+
try {
|
|
228
|
+
const rs = await target.batch(unwrap);
|
|
229
|
+
const each = (performance.now() - t0) / Math.max(1, rs.length);
|
|
230
|
+
rs.forEach((r, i) => d1Point(sqls[i] ?? '', 'batch', each, r));
|
|
231
|
+
return rs;
|
|
232
|
+
}
|
|
233
|
+
catch (err) {
|
|
234
|
+
const each = (performance.now() - t0) / Math.max(1, sqls.length);
|
|
235
|
+
for (const sql of sqls)
|
|
236
|
+
d1Point(sql, 'batch', each);
|
|
237
|
+
throw err;
|
|
238
|
+
}
|
|
239
|
+
};
|
|
240
|
+
const v = Reflect.get(target, prop, receiver);
|
|
241
|
+
return typeof v === 'function' ? v.bind(target) : v;
|
|
242
|
+
},
|
|
243
|
+
});
|
|
244
|
+
}
|
|
245
|
+
const STATS_RE = /"statistics"\s*:\s*\{\s*"elapsed"\s*:\s*([\d.e+-]+)\s*,\s*"rows_read"\s*:\s*(\d+)\s*,\s*"bytes_read"\s*:\s*(\d+)/;
|
|
246
|
+
const ROWS_RE = /"rows"\s*:\s*(\d+)\s*,/g;
|
|
247
|
+
/** `X-ClickHouse-Summary`: what the server knew when it sent the headers. */
|
|
248
|
+
function summaryOf(res) {
|
|
249
|
+
try {
|
|
250
|
+
const s = JSON.parse(res.headers.get('x-clickhouse-summary') ?? '{}');
|
|
251
|
+
return {
|
|
252
|
+
rowsRead: num(s.read_rows),
|
|
253
|
+
bytesRead: num(s.read_bytes),
|
|
254
|
+
rowsWritten: num(s.written_rows),
|
|
255
|
+
rowsOut: s.result_rows === undefined ? undefined : num(s.result_rows),
|
|
256
|
+
engineMs: s.elapsed_ns === undefined ? undefined : num(s.elapsed_ns) / 1e6,
|
|
257
|
+
};
|
|
258
|
+
}
|
|
259
|
+
catch {
|
|
260
|
+
return {};
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
const sqlOf = (input, init) => {
|
|
264
|
+
if (typeof init?.body === 'string')
|
|
265
|
+
return init.body;
|
|
266
|
+
try {
|
|
267
|
+
return new URL(input instanceof Request ? input.url : String(input)).searchParams.get('query');
|
|
268
|
+
}
|
|
269
|
+
catch {
|
|
270
|
+
return null;
|
|
271
|
+
}
|
|
272
|
+
};
|
|
273
|
+
/**
|
|
274
|
+
* The ClickHouse VPC binding, metered. The point is written when the caller has read the whole
|
|
275
|
+
* answer: wall time to the last byte, and the `statistics` ClickHouse puts at the end of a JSON
|
|
276
|
+
* answer (rows and bytes read, its own time), else the summary header.
|
|
277
|
+
*/
|
|
278
|
+
export function meteredClickHouse(binding) {
|
|
279
|
+
if (!binding)
|
|
280
|
+
return binding;
|
|
281
|
+
const fetchMetered = async (input, init) => {
|
|
282
|
+
const sql = sqlOf(input, init);
|
|
283
|
+
if (!sql)
|
|
284
|
+
return binding.fetch(input, init);
|
|
285
|
+
const method = /^\s*INSERT\b/i.test(sql) ? 'insert' : 'query';
|
|
286
|
+
const t0 = performance.now();
|
|
287
|
+
const caller = currentCaller();
|
|
288
|
+
let res;
|
|
289
|
+
try {
|
|
290
|
+
res = await binding.fetch(input, init);
|
|
291
|
+
}
|
|
292
|
+
catch (err) {
|
|
293
|
+
meterQuery({
|
|
294
|
+
engine: 'clickhouse',
|
|
295
|
+
sql,
|
|
296
|
+
method,
|
|
297
|
+
ok: false,
|
|
298
|
+
ms: performance.now() - t0,
|
|
299
|
+
caller,
|
|
300
|
+
});
|
|
301
|
+
throw err;
|
|
302
|
+
}
|
|
303
|
+
const head = summaryOf(res);
|
|
304
|
+
if (!res.ok || !res.body) {
|
|
305
|
+
meterQuery({
|
|
306
|
+
engine: 'clickhouse',
|
|
307
|
+
sql,
|
|
308
|
+
method,
|
|
309
|
+
ok: res.ok,
|
|
310
|
+
ms: performance.now() - t0,
|
|
311
|
+
status: res.status,
|
|
312
|
+
caller,
|
|
313
|
+
...head,
|
|
314
|
+
});
|
|
315
|
+
return res;
|
|
316
|
+
}
|
|
317
|
+
let bytes = 0;
|
|
318
|
+
let tail = new Uint8Array(0);
|
|
319
|
+
const counted = res.body.pipeThrough(new TransformStream({
|
|
320
|
+
transform(chunk, ctl) {
|
|
321
|
+
bytes += chunk.byteLength;
|
|
322
|
+
// Keep the last 1 KB: the answer's `rows` and `statistics` close it.
|
|
323
|
+
const joined = new Uint8Array(tail.byteLength + chunk.byteLength);
|
|
324
|
+
joined.set(tail, 0);
|
|
325
|
+
joined.set(chunk, tail.byteLength);
|
|
326
|
+
tail = joined.subarray(Math.max(0, joined.byteLength - 1024));
|
|
327
|
+
ctl.enqueue(chunk);
|
|
328
|
+
},
|
|
329
|
+
flush() {
|
|
330
|
+
const text = new TextDecoder().decode(tail);
|
|
331
|
+
const st = text.match(STATS_RE);
|
|
332
|
+
const rows = [...text.matchAll(ROWS_RE)].pop();
|
|
333
|
+
meterQuery({
|
|
334
|
+
engine: 'clickhouse',
|
|
335
|
+
sql,
|
|
336
|
+
method,
|
|
337
|
+
ok: true,
|
|
338
|
+
ms: performance.now() - t0,
|
|
339
|
+
status: res.status,
|
|
340
|
+
bytesOut: bytes,
|
|
341
|
+
caller,
|
|
342
|
+
...head,
|
|
343
|
+
...(rows ? { rowsOut: num(rows[1]) } : {}),
|
|
344
|
+
...(st
|
|
345
|
+
? { engineMs: num(st[1]) * 1000, rowsRead: num(st[2]), bytesRead: num(st[3]) }
|
|
346
|
+
: {}),
|
|
347
|
+
});
|
|
348
|
+
},
|
|
349
|
+
}));
|
|
350
|
+
return new Response(counted, {
|
|
351
|
+
status: res.status,
|
|
352
|
+
statusText: res.statusText,
|
|
353
|
+
headers: res.headers,
|
|
354
|
+
});
|
|
355
|
+
};
|
|
356
|
+
return new Proxy(binding, {
|
|
357
|
+
get(target, prop, receiver) {
|
|
358
|
+
if (prop === 'fetch')
|
|
359
|
+
return fetchMetered;
|
|
360
|
+
const v = Reflect.get(target, prop, receiver);
|
|
361
|
+
return typeof v === 'function' ? v.bind(target) : v;
|
|
362
|
+
},
|
|
363
|
+
});
|
|
364
|
+
}
|
|
365
|
+
/**
|
|
366
|
+
* A Durable Object's `storage.sql`, metered for `caller`. Each statement is read to the end at
|
|
367
|
+
* `exec`, so its rows read and written are known; callers read rows only through `toArray`.
|
|
368
|
+
*/
|
|
369
|
+
export function meteredSql(sql, caller) {
|
|
370
|
+
const native = sql;
|
|
371
|
+
return new Proxy(native, {
|
|
372
|
+
get(target, prop, receiver) {
|
|
373
|
+
if (prop === 'exec')
|
|
374
|
+
return (query, ...bindings) => {
|
|
375
|
+
const t0 = performance.now();
|
|
376
|
+
try {
|
|
377
|
+
const cursor = target.exec(query, ...bindings);
|
|
378
|
+
const rows = cursor.toArray();
|
|
379
|
+
meterQuery({
|
|
380
|
+
engine: 'do_sqlite',
|
|
381
|
+
sql: query,
|
|
382
|
+
method: 'exec',
|
|
383
|
+
ok: true,
|
|
384
|
+
ms: performance.now() - t0,
|
|
385
|
+
rowsRead: cursor.rowsRead,
|
|
386
|
+
rowsWritten: cursor.rowsWritten,
|
|
387
|
+
rowsOut: rows.length,
|
|
388
|
+
caller,
|
|
389
|
+
});
|
|
390
|
+
return {
|
|
391
|
+
toArray: () => rows,
|
|
392
|
+
rowsRead: cursor.rowsRead,
|
|
393
|
+
rowsWritten: cursor.rowsWritten,
|
|
394
|
+
};
|
|
395
|
+
}
|
|
396
|
+
catch (err) {
|
|
397
|
+
meterQuery({
|
|
398
|
+
engine: 'do_sqlite',
|
|
399
|
+
sql: query,
|
|
400
|
+
method: 'exec',
|
|
401
|
+
ok: false,
|
|
402
|
+
ms: performance.now() - t0,
|
|
403
|
+
caller,
|
|
404
|
+
});
|
|
405
|
+
throw err;
|
|
406
|
+
}
|
|
407
|
+
};
|
|
408
|
+
const v = Reflect.get(target, prop, receiver);
|
|
409
|
+
return typeof v === 'function' ? v.bind(target) : v;
|
|
410
|
+
},
|
|
411
|
+
});
|
|
412
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,43 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@columenlabs/lumen-ops",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.1.35",
|
|
4
|
+
"description": "Lumen's ops meters: every data query an app runs as one Analytics Engine point, by its shape only.",
|
|
5
|
+
"license": "MIT",
|
|
6
|
+
"type": "module",
|
|
7
|
+
"main": "./dist/index.js",
|
|
8
|
+
"types": "./dist/index.d.ts",
|
|
9
|
+
"exports": {
|
|
10
|
+
".": {
|
|
11
|
+
"types": "./dist/index.d.ts",
|
|
12
|
+
"default": "./dist/index.js"
|
|
13
|
+
},
|
|
14
|
+
"./points": {
|
|
15
|
+
"types": "./dist/points.d.ts",
|
|
16
|
+
"default": "./dist/points.js"
|
|
17
|
+
},
|
|
18
|
+
"./queries": {
|
|
19
|
+
"types": "./dist/queries.d.ts",
|
|
20
|
+
"default": "./dist/queries.js"
|
|
21
|
+
}
|
|
22
|
+
},
|
|
23
|
+
"files": [
|
|
24
|
+
"dist",
|
|
25
|
+
"LICENSE",
|
|
26
|
+
"NOTICE"
|
|
27
|
+
],
|
|
28
|
+
"sideEffects": [
|
|
29
|
+
"**/*.css"
|
|
30
|
+
],
|
|
31
|
+
"homepage": "https://github.com/columenlabs/lumen",
|
|
32
|
+
"repository": {
|
|
33
|
+
"type": "git",
|
|
34
|
+
"url": "git+https://github.com/columenlabs/lumen.git",
|
|
35
|
+
"directory": "packages/ops"
|
|
36
|
+
},
|
|
37
|
+
"bugs": "https://github.com/columenlabs/lumen/issues",
|
|
38
|
+
"publishConfig": {
|
|
39
|
+
"access": "public",
|
|
40
|
+
"registry": "https://registry.npmjs.org/"
|
|
41
|
+
},
|
|
42
|
+
"dependencies": {}
|
|
43
|
+
}
|