@knowledgefutures/sdk 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -0
- package/dist/whistle/hono.d.ts +42 -0
- package/dist/whistle/hono.js +89 -0
- package/dist/whistle/index.d.ts +135 -0
- package/dist/whistle/index.js +249 -0
- package/dist/whistle/wire.d.ts +64 -0
- package/dist/whistle/wire.js +98 -0
- package/package.json +5 -1
- package/dist/analytics/beacon.d.ts +0 -25
- package/dist/analytics/beacon.js +0 -86
- package/dist/analytics/binding.d.ts +0 -24
- package/dist/analytics/binding.js +0 -46
- package/dist/analytics/collector.d.ts +0 -10
- package/dist/analytics/collector.js +0 -10
- package/dist/analytics/cubes.d.ts +0 -96
- package/dist/analytics/cubes.js +0 -213
- package/dist/analytics/envelope.d.ts +0 -135
- package/dist/analytics/envelope.js +0 -231
- package/dist/analytics/errors.d.ts +0 -24
- package/dist/analytics/errors.js +0 -20
- package/dist/analytics/index.d.ts +0 -38
- package/dist/analytics/index.js +0 -98
- package/dist/analytics/memory.d.ts +0 -35
- package/dist/analytics/memory.js +0 -66
- package/dist/analytics/testing.d.ts +0 -6
- package/dist/analytics/testing.js +0 -6
- package/dist/analytics/types.d.ts +0 -22
- package/dist/analytics/types.js +0 -1
- package/dist/storage/conformance.d.ts +0 -24
- package/dist/storage/conformance.js +0 -311
- package/dist/storage/local.d.ts +0 -26
- package/dist/storage/local.js +0 -57
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The wire format: one JSON object per event, the same on every runtime.
|
|
3
|
+
*
|
|
4
|
+
* Sent to Whistle's `/v1/events` as NDJSON, or written as the body of a
|
|
5
|
+
* `console.log`/`console.error` line. Box agents send the same objects.
|
|
6
|
+
* Whistle parses it with `parseWire`, so this file is the contract between
|
|
7
|
+
* apps and Whistle; kf-whistle's `packages/telemetry/src/wire.ts` must say
|
|
8
|
+
* the same until Whistle imports it from here.
|
|
9
|
+
*/
|
|
10
|
+
/** Security events: what a careful attacker can't avoid causing. Add kinds here, nowhere else. */
|
|
11
|
+
export const SECURITY_KINDS = [
|
|
12
|
+
'auth.failed',
|
|
13
|
+
'auth.success',
|
|
14
|
+
'auth.mfa_failed',
|
|
15
|
+
'auth.reset_requested',
|
|
16
|
+
'token.created',
|
|
17
|
+
'token.used_new_ip',
|
|
18
|
+
'access.denied',
|
|
19
|
+
'admin.action',
|
|
20
|
+
'secret.read',
|
|
21
|
+
'member.added',
|
|
22
|
+
'export.large',
|
|
23
|
+
// From box agents (vector/vector.toml), not app code.
|
|
24
|
+
'ssh.failed',
|
|
25
|
+
'ssh.success',
|
|
26
|
+
'sudo',
|
|
27
|
+
];
|
|
28
|
+
const MAX_DETAIL_BYTES = 8_000;
|
|
29
|
+
const SENSITIVE_KEY = /pass(word)?|secret|token|cookie|authorization|api[-_]?key|session/i;
|
|
30
|
+
/**
|
|
31
|
+
* `detail` without anything that looks like a credential, and bounded in size.
|
|
32
|
+
* Apps pass objects they didn't fully build (a request's params, an error's
|
|
33
|
+
* cause), so this is the last line before a secret ends up in the lake.
|
|
34
|
+
*/
|
|
35
|
+
export function scrub(detail) {
|
|
36
|
+
if (!detail)
|
|
37
|
+
return undefined;
|
|
38
|
+
const walk = (value, depth) => {
|
|
39
|
+
if (depth > 4)
|
|
40
|
+
return '[depth]';
|
|
41
|
+
if (Array.isArray(value))
|
|
42
|
+
return value.slice(0, 50).map((v) => walk(v, depth + 1));
|
|
43
|
+
if (value && typeof value === 'object') {
|
|
44
|
+
const out = {};
|
|
45
|
+
for (const [k, v] of Object.entries(value)) {
|
|
46
|
+
out[k] = SENSITIVE_KEY.test(k) ? '[redacted]' : walk(v, depth + 1);
|
|
47
|
+
}
|
|
48
|
+
return out;
|
|
49
|
+
}
|
|
50
|
+
if (typeof value === 'string' && value.length > 2_000)
|
|
51
|
+
return `${value.slice(0, 2_000)}…`;
|
|
52
|
+
// JSON.stringify throws on a bigint.
|
|
53
|
+
if (typeof value === 'bigint')
|
|
54
|
+
return value.toString();
|
|
55
|
+
return value;
|
|
56
|
+
};
|
|
57
|
+
const out = walk(detail, 0);
|
|
58
|
+
if (JSON.stringify(out).length > MAX_DETAIL_BYTES)
|
|
59
|
+
return { truncated: true };
|
|
60
|
+
return out;
|
|
61
|
+
}
|
|
62
|
+
const isString = (v) => typeof v === 'string' && v.length > 0;
|
|
63
|
+
const KINDS = new Set(SECURITY_KINDS);
|
|
64
|
+
/**
|
|
65
|
+
* A wire event from a log body (a JSON string or an already-parsed object), or
|
|
66
|
+
* null when it isn't one. Tolerant of extra fields; strict about the ones rules
|
|
67
|
+
* depend on, so a malformed event never matches a rule by accident.
|
|
68
|
+
*/
|
|
69
|
+
export function parseWire(input) {
|
|
70
|
+
let value = input;
|
|
71
|
+
if (typeof value === 'string') {
|
|
72
|
+
const s = value.trim();
|
|
73
|
+
if (!s.startsWith('{') || !s.includes('"kf"'))
|
|
74
|
+
return null;
|
|
75
|
+
try {
|
|
76
|
+
value = JSON.parse(s);
|
|
77
|
+
}
|
|
78
|
+
catch {
|
|
79
|
+
return null;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
if (!value || typeof value !== 'object')
|
|
83
|
+
return null;
|
|
84
|
+
const o = value;
|
|
85
|
+
if (o.v !== 1 || !isString(o.project) || !isString(o.environment) || !isString(o.at))
|
|
86
|
+
return null;
|
|
87
|
+
if (o.kf === 'security') {
|
|
88
|
+
if (!isString(o.kind) || !KINDS.has(o.kind))
|
|
89
|
+
return null;
|
|
90
|
+
return o;
|
|
91
|
+
}
|
|
92
|
+
if (o.kf === 'error') {
|
|
93
|
+
if (!isString(o.type) || typeof o.message !== 'string')
|
|
94
|
+
return null;
|
|
95
|
+
return o;
|
|
96
|
+
}
|
|
97
|
+
return null;
|
|
98
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@knowledgefutures/sdk",
|
|
3
|
-
"version": "0.1
|
|
3
|
+
"version": "0.2.1",
|
|
4
4
|
"description": "Zero-config clients for Knowledge Futures platform services, configured by the console.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -21,6 +21,10 @@
|
|
|
21
21
|
"./storage/testing": {
|
|
22
22
|
"types": "./dist/storage/testing.d.ts",
|
|
23
23
|
"default": "./dist/storage/testing.js"
|
|
24
|
+
},
|
|
25
|
+
"./whistle": {
|
|
26
|
+
"types": "./dist/whistle/index.d.ts",
|
|
27
|
+
"default": "./dist/whistle/index.js"
|
|
24
28
|
}
|
|
25
29
|
},
|
|
26
30
|
"publishConfig": {
|
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* The page script, served by the collector at `/v1/b.js`. Every project loads
|
|
3
|
-
* the same line:
|
|
4
|
-
*
|
|
5
|
-
* <script src="https://<collector>/v1/b.js" defer></script>
|
|
6
|
-
*
|
|
7
|
-
* and a project whose domains the console can't list (PubPub's custom
|
|
8
|
-
* domains) adds `data-site="<site id>"`. Ported from the mitp-oecs beacon;
|
|
9
|
-
* three changes: it posts to wherever it was loaded from, it sends
|
|
10
|
-
* `text/plain` (so a cross-origin `sendBeacon` needs no preflight and still
|
|
11
|
-
* carries `Origin`, which is how the collector attributes it), and it reads
|
|
12
|
-
* `kf:scope`.
|
|
13
|
-
*
|
|
14
|
-
* What a page declares, the beacon reads:
|
|
15
|
-
*
|
|
16
|
-
* <meta name="kf:subject" content="pub:za65x7nh">
|
|
17
|
-
* <meta name="kf:subject-uri" content="https://doi.org/10.21428/…">
|
|
18
|
-
* <meta name="kf:scope" content="<community id>">
|
|
19
|
-
*
|
|
20
|
-
* One view per page load; `data-kf-event="download:pdf"` on any element
|
|
21
|
-
* reports that event on click; `window.kf.track(kind, name, props)` for the
|
|
22
|
-
* rest. No cookies, no storage, nothing about the reader beyond the referrer.
|
|
23
|
-
* Every failure is swallowed: analytics must never break a page.
|
|
24
|
-
*/
|
|
25
|
-
export declare const BEACON_JS = "(() => {\n try {\n const script = document.currentScript;\n if (!script || !script.src) return;\n const endpoint = new URL('e', script.src);\n const site = script.getAttribute('data-site');\n if (site) endpoint.searchParams.set('site', site);\n const EP = endpoint.toString();\n const meta = (n) => {\n const el = document.querySelector('meta[name=\"' + n + '\"]');\n return (el && el.getAttribute('content')) || undefined;\n };\n let utm;\n new URLSearchParams(location.search).forEach((v, k) => {\n const key = k.indexOf('utm_') === 0 ? k.slice(4) : '';\n if (v && ['source', 'medium', 'campaign', 'term', 'content'].indexOf(key) >= 0) {\n (utm = utm || {})[key] = String(v);\n }\n });\n const strs = (o) => {\n if (!o || typeof o !== 'object') return undefined;\n const out = {};\n let n = 0;\n for (const k in o) if (o[k] !== undefined && o[k] !== null) { out[k] = String(o[k]); n++; }\n return n ? out : undefined;\n };\n const send = (kind, name, props) => {\n try {\n const body = {\n kind: kind,\n subject: meta('kf:subject'),\n subjectUri: meta('kf:subject-uri'),\n scope: meta('kf:scope'),\n path: location.pathname,\n referrer: document.referrer || undefined,\n name: (typeof name === 'string' && name) || undefined,\n props: strs(props),\n utm: utm,\n };\n for (const k in body) if (body[k] === undefined) delete body[k];\n const json = JSON.stringify(body);\n if (navigator.sendBeacon && navigator.sendBeacon(EP, json)) return;\n fetch(EP, { method: 'POST', mode: 'no-cors', keepalive: true, credentials: 'omit',\n headers: { 'content-type': 'text/plain' }, body: json });\n } catch (e) {}\n };\n window.kf = { track: send };\n const view = () => send('view');\n if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', view, { once: true });\n else view();\n document.addEventListener('click', (e) => {\n try {\n const el = e.target && e.target.closest && e.target.closest('[data-kf-event]');\n if (!el) return;\n const parts = (el.getAttribute('data-kf-event') || '').split(':');\n const kind = parts.shift();\n if (kind) send(kind, parts.join(':'));\n } catch (err) {}\n }, true);\n } catch (e) {}\n})();\n";
|
package/dist/analytics/beacon.js
DELETED
|
@@ -1,86 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* The page script, served by the collector at `/v1/b.js`. Every project loads
|
|
3
|
-
* the same line:
|
|
4
|
-
*
|
|
5
|
-
* <script src="https://<collector>/v1/b.js" defer></script>
|
|
6
|
-
*
|
|
7
|
-
* and a project whose domains the console can't list (PubPub's custom
|
|
8
|
-
* domains) adds `data-site="<site id>"`. Ported from the mitp-oecs beacon;
|
|
9
|
-
* three changes: it posts to wherever it was loaded from, it sends
|
|
10
|
-
* `text/plain` (so a cross-origin `sendBeacon` needs no preflight and still
|
|
11
|
-
* carries `Origin`, which is how the collector attributes it), and it reads
|
|
12
|
-
* `kf:scope`.
|
|
13
|
-
*
|
|
14
|
-
* What a page declares, the beacon reads:
|
|
15
|
-
*
|
|
16
|
-
* <meta name="kf:subject" content="pub:za65x7nh">
|
|
17
|
-
* <meta name="kf:subject-uri" content="https://doi.org/10.21428/…">
|
|
18
|
-
* <meta name="kf:scope" content="<community id>">
|
|
19
|
-
*
|
|
20
|
-
* One view per page load; `data-kf-event="download:pdf"` on any element
|
|
21
|
-
* reports that event on click; `window.kf.track(kind, name, props)` for the
|
|
22
|
-
* rest. No cookies, no storage, nothing about the reader beyond the referrer.
|
|
23
|
-
* Every failure is swallowed: analytics must never break a page.
|
|
24
|
-
*/
|
|
25
|
-
export const BEACON_JS = `(() => {
|
|
26
|
-
try {
|
|
27
|
-
const script = document.currentScript;
|
|
28
|
-
if (!script || !script.src) return;
|
|
29
|
-
const endpoint = new URL('e', script.src);
|
|
30
|
-
const site = script.getAttribute('data-site');
|
|
31
|
-
if (site) endpoint.searchParams.set('site', site);
|
|
32
|
-
const EP = endpoint.toString();
|
|
33
|
-
const meta = (n) => {
|
|
34
|
-
const el = document.querySelector('meta[name="' + n + '"]');
|
|
35
|
-
return (el && el.getAttribute('content')) || undefined;
|
|
36
|
-
};
|
|
37
|
-
let utm;
|
|
38
|
-
new URLSearchParams(location.search).forEach((v, k) => {
|
|
39
|
-
const key = k.indexOf('utm_') === 0 ? k.slice(4) : '';
|
|
40
|
-
if (v && ['source', 'medium', 'campaign', 'term', 'content'].indexOf(key) >= 0) {
|
|
41
|
-
(utm = utm || {})[key] = String(v);
|
|
42
|
-
}
|
|
43
|
-
});
|
|
44
|
-
const strs = (o) => {
|
|
45
|
-
if (!o || typeof o !== 'object') return undefined;
|
|
46
|
-
const out = {};
|
|
47
|
-
let n = 0;
|
|
48
|
-
for (const k in o) if (o[k] !== undefined && o[k] !== null) { out[k] = String(o[k]); n++; }
|
|
49
|
-
return n ? out : undefined;
|
|
50
|
-
};
|
|
51
|
-
const send = (kind, name, props) => {
|
|
52
|
-
try {
|
|
53
|
-
const body = {
|
|
54
|
-
kind: kind,
|
|
55
|
-
subject: meta('kf:subject'),
|
|
56
|
-
subjectUri: meta('kf:subject-uri'),
|
|
57
|
-
scope: meta('kf:scope'),
|
|
58
|
-
path: location.pathname,
|
|
59
|
-
referrer: document.referrer || undefined,
|
|
60
|
-
name: (typeof name === 'string' && name) || undefined,
|
|
61
|
-
props: strs(props),
|
|
62
|
-
utm: utm,
|
|
63
|
-
};
|
|
64
|
-
for (const k in body) if (body[k] === undefined) delete body[k];
|
|
65
|
-
const json = JSON.stringify(body);
|
|
66
|
-
if (navigator.sendBeacon && navigator.sendBeacon(EP, json)) return;
|
|
67
|
-
fetch(EP, { method: 'POST', mode: 'no-cors', keepalive: true, credentials: 'omit',
|
|
68
|
-
headers: { 'content-type': 'text/plain' }, body: json });
|
|
69
|
-
} catch (e) {}
|
|
70
|
-
};
|
|
71
|
-
window.kf = { track: send };
|
|
72
|
-
const view = () => send('view');
|
|
73
|
-
if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', view, { once: true });
|
|
74
|
-
else view();
|
|
75
|
-
document.addEventListener('click', (e) => {
|
|
76
|
-
try {
|
|
77
|
-
const el = e.target && e.target.closest && e.target.closest('[data-kf-event]');
|
|
78
|
-
if (!el) return;
|
|
79
|
-
const parts = (el.getAttribute('data-kf-event') || '').split(':');
|
|
80
|
-
const kind = parts.shift();
|
|
81
|
-
if (kind) send(kind, parts.join(':'));
|
|
82
|
-
} catch (err) {}
|
|
83
|
-
}, true);
|
|
84
|
-
} catch (e) {}
|
|
85
|
-
})();
|
|
86
|
-
`;
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* `KF_ANALYTICS_URL` — the whole contract between the console, which
|
|
3
|
-
* registers a site with the collector and writes the binding into a
|
|
4
|
-
* project's environment, and the SDK, which reads it. Both import this file,
|
|
5
|
-
* so the format has exactly one definition.
|
|
6
|
-
*
|
|
7
|
-
* https://<site>:<secret>@<collector host>[/<base path>]
|
|
8
|
-
*
|
|
9
|
-
* `http` only for a local collector. The site id is public (a keyed beacon
|
|
10
|
-
* carries it); the secret signs server events and queries, and never reaches
|
|
11
|
-
* a browser. `log:` is `kf dev`'s binding: events are printed, nothing is
|
|
12
|
-
* sent.
|
|
13
|
-
*/
|
|
14
|
-
export interface AnalyticsBinding {
|
|
15
|
-
/** The collector's origin and base path, no trailing slash. */
|
|
16
|
-
endpoint: string;
|
|
17
|
-
site: string;
|
|
18
|
-
secret: string;
|
|
19
|
-
}
|
|
20
|
-
export declare const ANALYTICS_ENV = "KF_ANALYTICS_URL";
|
|
21
|
-
export declare const LOG_BINDING = "log:";
|
|
22
|
-
export declare function encodeAnalyticsBinding(b: AnalyticsBinding): string;
|
|
23
|
-
/** The binding, or `'log'` for `kf dev`'s print-only binding. */
|
|
24
|
-
export declare function decodeAnalyticsBinding(value: string): AnalyticsBinding | 'log';
|
|
@@ -1,46 +0,0 @@
|
|
|
1
|
-
import { AnalyticsError } from './errors.js';
|
|
2
|
-
export const ANALYTICS_ENV = 'KF_ANALYTICS_URL';
|
|
3
|
-
export const LOG_BINDING = 'log:';
|
|
4
|
-
const SITE = /^[A-Za-z0-9_-]{1,64}$/;
|
|
5
|
-
export function encodeAnalyticsBinding(b) {
|
|
6
|
-
const endpoint = new URL(b.endpoint);
|
|
7
|
-
if (endpoint.protocol !== 'https:' && endpoint.protocol !== 'http:') {
|
|
8
|
-
throw new AnalyticsError('invalid', `endpoint must be http(s): ${b.endpoint}`);
|
|
9
|
-
}
|
|
10
|
-
if (!SITE.test(b.site))
|
|
11
|
-
throw new AnalyticsError('invalid', `not a site id: ${b.site}`);
|
|
12
|
-
if (!b.secret)
|
|
13
|
-
throw new AnalyticsError('invalid', 'the secret is empty');
|
|
14
|
-
const path = endpoint.pathname.replace(/\/+$/, '');
|
|
15
|
-
return `${endpoint.protocol}//${b.site}:${encodeURIComponent(b.secret)}@${endpoint.host}${path}`;
|
|
16
|
-
}
|
|
17
|
-
/** The binding, or `'log'` for `kf dev`'s print-only binding. */
|
|
18
|
-
export function decodeAnalyticsBinding(value) {
|
|
19
|
-
const v = value.trim();
|
|
20
|
-
if (v === LOG_BINDING)
|
|
21
|
-
return 'log';
|
|
22
|
-
let url;
|
|
23
|
-
try {
|
|
24
|
-
url = new URL(v);
|
|
25
|
-
}
|
|
26
|
-
catch {
|
|
27
|
-
throw bad('not a URL');
|
|
28
|
-
}
|
|
29
|
-
if (url.protocol !== 'https:' && url.protocol !== 'http:') {
|
|
30
|
-
throw bad(`the scheme is https (or http for a local collector), or the value is "${LOG_BINDING}"`);
|
|
31
|
-
}
|
|
32
|
-
const site = decodeURIComponent(url.username);
|
|
33
|
-
if (!SITE.test(site))
|
|
34
|
-
throw bad('the user part is the site id');
|
|
35
|
-
if (!url.password)
|
|
36
|
-
throw bad('the secret is missing');
|
|
37
|
-
return {
|
|
38
|
-
endpoint: `${url.protocol}//${url.host}${url.pathname.replace(/\/+$/, '')}`,
|
|
39
|
-
site,
|
|
40
|
-
secret: decodeURIComponent(url.password),
|
|
41
|
-
};
|
|
42
|
-
}
|
|
43
|
-
function bad(why) {
|
|
44
|
-
// Never echo the value: it carries the secret.
|
|
45
|
-
return new AnalyticsError('not_configured', `${ANALYTICS_ENV} is malformed: ${why}`);
|
|
46
|
-
}
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* `@knowledgefutures/sdk/analytics/collector` — what the `kf/analytics`
|
|
3
|
-
* collector imports: the event format, the query semantics and the page
|
|
4
|
-
* script. The SDK's client half uses the same files, which is the point:
|
|
5
|
-
* a hint the SDK accepts is a hint the collector accepts, and a query answers
|
|
6
|
-
* the same from the memory double as from the service.
|
|
7
|
-
*/
|
|
8
|
-
export { buildRow, device, FIELDS, KINDS, LIMITS, SCHEMA, UTM_KEYS, validateHint, type Device, type EventRow, type Hint, type Kind, type RowContext, type ServerHint, type Source, } from './envelope.js';
|
|
9
|
-
export { CUBES, cubeRows, MAX_ROWS, runQuery, validateQuery, type Cube, type CubeRow, type Query, type QueryResult, } from './cubes.js';
|
|
10
|
-
export { BEACON_JS } from './beacon.js';
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* `@knowledgefutures/sdk/analytics/collector` — what the `kf/analytics`
|
|
3
|
-
* collector imports: the event format, the query semantics and the page
|
|
4
|
-
* script. The SDK's client half uses the same files, which is the point:
|
|
5
|
-
* a hint the SDK accepts is a hint the collector accepts, and a query answers
|
|
6
|
-
* the same from the memory double as from the service.
|
|
7
|
-
*/
|
|
8
|
-
export { buildRow, device, FIELDS, KINDS, LIMITS, SCHEMA, UTM_KEYS, validateHint, } from './envelope.js';
|
|
9
|
-
export { CUBES, cubeRows, MAX_ROWS, runQuery, validateQuery, } from './cubes.js';
|
|
10
|
-
export { BEACON_JS } from './beacon.js';
|
|
@@ -1,96 +0,0 @@
|
|
|
1
|
-
import type { EventRow } from './envelope.js';
|
|
2
|
-
/**
|
|
3
|
-
* The cubes: one site's events, collapsed into daily rows the compactor
|
|
4
|
-
* writes and every query reads. They are OECS's eleven summary files
|
|
5
|
-
* generalized — `pubs` and `collections` became `subjects`, keyed by
|
|
6
|
-
* `subject_type`, and every cube gains `scope`.
|
|
7
|
-
*
|
|
8
|
-
* Two rules hold everywhere:
|
|
9
|
-
*
|
|
10
|
-
* - **Bots are gone** (`device = 'bot'`) before anything is counted.
|
|
11
|
-
* - **Every metric is a count, so it sums across days.** A query over any
|
|
12
|
-
* range is a sum of daily rows, and exact. A metric that can't be summed —
|
|
13
|
-
* distinct subjects in a month, unique people — can never be a column.
|
|
14
|
-
*
|
|
15
|
-
* `views` are rows with `kind = 'view'`; `visits` are views with
|
|
16
|
-
* `is_unique` (the referrer heuristic: visits, never "unique visitors").
|
|
17
|
-
*
|
|
18
|
-
* `cubeRows` is the reference implementation, used by the memory double.
|
|
19
|
-
* The compactor computes the same rows in SQL, and the conformance suite
|
|
20
|
-
* holds the two to each other.
|
|
21
|
-
*/
|
|
22
|
-
export declare const CUBES: {
|
|
23
|
-
readonly days: {
|
|
24
|
-
readonly dims: readonly ["day", "scope"];
|
|
25
|
-
readonly metrics: readonly ["views", "visits", "downloads", "searches", "custom"];
|
|
26
|
-
};
|
|
27
|
-
readonly subjects: {
|
|
28
|
-
readonly dims: readonly ["day", "scope", "subject_type", "subject_id"];
|
|
29
|
-
readonly metrics: readonly ["views", "visits", "downloads"];
|
|
30
|
-
};
|
|
31
|
-
readonly pages: {
|
|
32
|
-
readonly dims: readonly ["day", "scope", "path"];
|
|
33
|
-
readonly metrics: readonly ["views", "visits"];
|
|
34
|
-
};
|
|
35
|
-
readonly referrers: {
|
|
36
|
-
readonly dims: readonly ["day", "scope", "referrer_host"];
|
|
37
|
-
readonly metrics: readonly ["views", "visits"];
|
|
38
|
-
};
|
|
39
|
-
readonly campaigns: {
|
|
40
|
-
readonly dims: readonly ["day", "scope", "utm_source", "utm_medium", "utm_campaign"];
|
|
41
|
-
readonly metrics: readonly ["views", "visits"];
|
|
42
|
-
};
|
|
43
|
-
readonly devices: {
|
|
44
|
-
readonly dims: readonly ["day", "scope", "device"];
|
|
45
|
-
readonly metrics: readonly ["views", "visits"];
|
|
46
|
-
};
|
|
47
|
-
readonly countries: {
|
|
48
|
-
readonly dims: readonly ["day", "scope", "country"];
|
|
49
|
-
readonly metrics: readonly ["views", "visits"];
|
|
50
|
-
};
|
|
51
|
-
readonly downloads: {
|
|
52
|
-
readonly dims: readonly ["day", "scope", "subject_type", "subject_id", "name"];
|
|
53
|
-
readonly metrics: readonly ["downloads"];
|
|
54
|
-
};
|
|
55
|
-
readonly searches: {
|
|
56
|
-
readonly dims: readonly ["day", "scope", "name"];
|
|
57
|
-
readonly metrics: readonly ["searches"];
|
|
58
|
-
};
|
|
59
|
-
};
|
|
60
|
-
export type Cube = keyof typeof CUBES;
|
|
61
|
-
export type Dim<C extends Cube = Cube> = (typeof CUBES)[C]['dims'][number];
|
|
62
|
-
export type Metric<C extends Cube = Cube> = (typeof CUBES)[C]['metrics'][number];
|
|
63
|
-
/** A cube row: its dimensions (null where the event had none) and its counts. */
|
|
64
|
-
export type CubeRow = Record<string, string | number | null>;
|
|
65
|
-
/** One cube's daily rows from raw events, sorted by day. The compactor's contract, in code. */
|
|
66
|
-
export declare function cubeRows(cube: Cube, events: readonly EventRow[]): CubeRow[];
|
|
67
|
-
export interface Query<C extends Cube = Cube> {
|
|
68
|
-
cube: C;
|
|
69
|
-
/** First closed UTC day, inclusive, `YYYY-MM-DD`. */
|
|
70
|
-
from: string;
|
|
71
|
-
/** Last closed UTC day, inclusive. */
|
|
72
|
-
to: string;
|
|
73
|
-
/** Equality on a dimension; an array is "any of". `null` matches rows without the dimension. */
|
|
74
|
-
filter?: Partial<Record<Dim<C>, string | null | ReadonlyArray<string | null>>>;
|
|
75
|
-
/** The dimensions to keep. Omitted or `[]`: one row of totals. */
|
|
76
|
-
groupBy?: ReadonlyArray<Dim<C>>;
|
|
77
|
-
/** A metric or dimension, `-` first for descending. Default: the group's dimensions, ascending. */
|
|
78
|
-
order?: string;
|
|
79
|
-
/** At most `MAX_ROWS`. */
|
|
80
|
-
limit?: number;
|
|
81
|
-
/** Another of the project's environments. Default: the caller's own. */
|
|
82
|
-
environment?: string;
|
|
83
|
-
}
|
|
84
|
-
export interface QueryResult {
|
|
85
|
-
rows: CubeRow[];
|
|
86
|
-
/** The last day the cubes cover, or null when the site has no data yet. Today is never in them. */
|
|
87
|
-
through: string | null;
|
|
88
|
-
}
|
|
89
|
-
export declare const MAX_ROWS = 10000;
|
|
90
|
-
/** Why a query is refused, or null. The collector runs this too, so both answer alike. */
|
|
91
|
-
export declare function validateQuery(q: unknown): string | null;
|
|
92
|
-
/**
|
|
93
|
-
* Answer a query from a cube's daily rows: keep the range and the filter,
|
|
94
|
-
* group, sum, sort, cut. Throws `invalid` for a query `validateQuery` refuses.
|
|
95
|
-
*/
|
|
96
|
-
export declare function runQuery(rows: readonly CubeRow[], q: Query): CubeRow[];
|
package/dist/analytics/cubes.js
DELETED
|
@@ -1,213 +0,0 @@
|
|
|
1
|
-
import { AnalyticsError } from './errors.js';
|
|
2
|
-
/**
|
|
3
|
-
* The cubes: one site's events, collapsed into daily rows the compactor
|
|
4
|
-
* writes and every query reads. They are OECS's eleven summary files
|
|
5
|
-
* generalized — `pubs` and `collections` became `subjects`, keyed by
|
|
6
|
-
* `subject_type`, and every cube gains `scope`.
|
|
7
|
-
*
|
|
8
|
-
* Two rules hold everywhere:
|
|
9
|
-
*
|
|
10
|
-
* - **Bots are gone** (`device = 'bot'`) before anything is counted.
|
|
11
|
-
* - **Every metric is a count, so it sums across days.** A query over any
|
|
12
|
-
* range is a sum of daily rows, and exact. A metric that can't be summed —
|
|
13
|
-
* distinct subjects in a month, unique people — can never be a column.
|
|
14
|
-
*
|
|
15
|
-
* `views` are rows with `kind = 'view'`; `visits` are views with
|
|
16
|
-
* `is_unique` (the referrer heuristic: visits, never "unique visitors").
|
|
17
|
-
*
|
|
18
|
-
* `cubeRows` is the reference implementation, used by the memory double.
|
|
19
|
-
* The compactor computes the same rows in SQL, and the conformance suite
|
|
20
|
-
* holds the two to each other.
|
|
21
|
-
*/
|
|
22
|
-
export const CUBES = {
|
|
23
|
-
days: { dims: ['day', 'scope'], metrics: ['views', 'visits', 'downloads', 'searches', 'custom'] },
|
|
24
|
-
subjects: {
|
|
25
|
-
dims: ['day', 'scope', 'subject_type', 'subject_id'],
|
|
26
|
-
metrics: ['views', 'visits', 'downloads'],
|
|
27
|
-
},
|
|
28
|
-
pages: { dims: ['day', 'scope', 'path'], metrics: ['views', 'visits'] },
|
|
29
|
-
referrers: { dims: ['day', 'scope', 'referrer_host'], metrics: ['views', 'visits'] },
|
|
30
|
-
campaigns: {
|
|
31
|
-
dims: ['day', 'scope', 'utm_source', 'utm_medium', 'utm_campaign'],
|
|
32
|
-
metrics: ['views', 'visits'],
|
|
33
|
-
},
|
|
34
|
-
devices: { dims: ['day', 'scope', 'device'], metrics: ['views', 'visits'] },
|
|
35
|
-
countries: { dims: ['day', 'scope', 'country'], metrics: ['views', 'visits'] },
|
|
36
|
-
downloads: {
|
|
37
|
-
dims: ['day', 'scope', 'subject_type', 'subject_id', 'name'],
|
|
38
|
-
metrics: ['downloads'],
|
|
39
|
-
},
|
|
40
|
-
searches: { dims: ['day', 'scope', 'name'], metrics: ['searches'] },
|
|
41
|
-
};
|
|
42
|
-
/** Which events a cube counts, beyond "not a bot". */
|
|
43
|
-
const KEEP = {
|
|
44
|
-
days: () => true,
|
|
45
|
-
subjects: (e) => e.subject_type !== null && (e.kind === 'view' || e.kind === 'download'),
|
|
46
|
-
pages: (e) => e.kind === 'view',
|
|
47
|
-
// External referrers only: a click from one page of the site to another is not a referral.
|
|
48
|
-
referrers: (e) => e.kind === 'view' && e.referrer_host !== null && e.referrer_host !== e.hostname,
|
|
49
|
-
campaigns: (e) => e.kind === 'view' &&
|
|
50
|
-
(e.utm_source !== null || e.utm_medium !== null || e.utm_campaign !== null),
|
|
51
|
-
devices: (e) => e.kind === 'view',
|
|
52
|
-
countries: (e) => e.kind === 'view',
|
|
53
|
-
downloads: (e) => e.kind === 'download',
|
|
54
|
-
searches: (e) => e.kind === 'search',
|
|
55
|
-
};
|
|
56
|
-
function counts(e) {
|
|
57
|
-
return {
|
|
58
|
-
views: e.kind === 'view' ? 1 : 0,
|
|
59
|
-
visits: e.kind === 'view' && e.is_unique ? 1 : 0,
|
|
60
|
-
downloads: e.kind === 'download' ? 1 : 0,
|
|
61
|
-
searches: e.kind === 'search' ? 1 : 0,
|
|
62
|
-
custom: e.kind === 'custom' ? 1 : 0,
|
|
63
|
-
};
|
|
64
|
-
}
|
|
65
|
-
/** One cube's daily rows from raw events, sorted by day. The compactor's contract, in code. */
|
|
66
|
-
export function cubeRows(cube, events) {
|
|
67
|
-
const { dims, metrics } = CUBES[cube];
|
|
68
|
-
const keep = KEEP[cube];
|
|
69
|
-
const rows = new Map();
|
|
70
|
-
for (const e of events) {
|
|
71
|
-
if (e.device === 'bot' || !keep(e))
|
|
72
|
-
continue;
|
|
73
|
-
const key = JSON.stringify(dims.map((d) => e[d]));
|
|
74
|
-
let row = rows.get(key);
|
|
75
|
-
if (!row) {
|
|
76
|
-
row = Object.fromEntries([
|
|
77
|
-
...dims.map((d) => [d, e[d]]),
|
|
78
|
-
...metrics.map((m) => [m, 0]),
|
|
79
|
-
]);
|
|
80
|
-
rows.set(key, row);
|
|
81
|
-
}
|
|
82
|
-
const c = counts(e);
|
|
83
|
-
for (const m of metrics)
|
|
84
|
-
row[m] = row[m] + c[m];
|
|
85
|
-
}
|
|
86
|
-
return [...rows.values()].sort((a, b) => String(a.day).localeCompare(String(b.day)));
|
|
87
|
-
}
|
|
88
|
-
export const MAX_ROWS = 10_000;
|
|
89
|
-
const DAY = /^\d{4}-\d{2}-\d{2}$/;
|
|
90
|
-
/** Why a query is refused, or null. The collector runs this too, so both answer alike. */
|
|
91
|
-
export function validateQuery(q) {
|
|
92
|
-
if (!q || typeof q !== 'object' || Array.isArray(q))
|
|
93
|
-
return 'query must be an object';
|
|
94
|
-
const query = q;
|
|
95
|
-
const known = new Set([
|
|
96
|
-
'cube',
|
|
97
|
-
'from',
|
|
98
|
-
'to',
|
|
99
|
-
'filter',
|
|
100
|
-
'groupBy',
|
|
101
|
-
'order',
|
|
102
|
-
'limit',
|
|
103
|
-
'environment',
|
|
104
|
-
]);
|
|
105
|
-
for (const key of Object.keys(query))
|
|
106
|
-
if (!known.has(key))
|
|
107
|
-
return `unknown key "${key}"`;
|
|
108
|
-
if (typeof query.cube !== 'string' || !Object.hasOwn(CUBES, query.cube)) {
|
|
109
|
-
return `cube must be one of ${Object.keys(CUBES).join(', ')}`;
|
|
110
|
-
}
|
|
111
|
-
const { dims, metrics } = CUBES[query.cube];
|
|
112
|
-
const isDim = (d) => dims.includes(d);
|
|
113
|
-
for (const key of ['from', 'to']) {
|
|
114
|
-
const v = query[key];
|
|
115
|
-
if (typeof v !== 'string' || !DAY.test(v) || Number.isNaN(Date.parse(v))) {
|
|
116
|
-
return `${key} must be a day, YYYY-MM-DD`;
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
|
-
if (query.from > query.to)
|
|
120
|
-
return 'from is after to';
|
|
121
|
-
if (query.filter !== undefined) {
|
|
122
|
-
if (!query.filter || typeof query.filter !== 'object' || Array.isArray(query.filter)) {
|
|
123
|
-
return 'filter must be an object';
|
|
124
|
-
}
|
|
125
|
-
for (const [d, v] of Object.entries(query.filter)) {
|
|
126
|
-
if (!isDim(d))
|
|
127
|
-
return `${query.cube} has no dimension "${d}"; it has ${dims.join(', ')}`;
|
|
128
|
-
const values = Array.isArray(v) ? v : [v];
|
|
129
|
-
if (values.some((x) => x !== null && typeof x !== 'string')) {
|
|
130
|
-
return `filter.${d} must be a string, null, or a list of them`;
|
|
131
|
-
}
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
if (query.groupBy !== undefined) {
|
|
135
|
-
if (!Array.isArray(query.groupBy))
|
|
136
|
-
return 'groupBy must be a list';
|
|
137
|
-
for (const d of query.groupBy) {
|
|
138
|
-
if (!isDim(d))
|
|
139
|
-
return `${query.cube} has no dimension "${String(d)}"; it has ${dims.join(', ')}`;
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
if (query.order !== undefined) {
|
|
143
|
-
const field = typeof query.order === 'string' ? query.order.replace(/^-/, '') : undefined;
|
|
144
|
-
const grouped = query.groupBy ?? [];
|
|
145
|
-
if (!field || !(metrics.includes(field) || grouped.includes(field))) {
|
|
146
|
-
return 'order must be a metric or a grouped dimension, "-" first for descending';
|
|
147
|
-
}
|
|
148
|
-
}
|
|
149
|
-
if (query.limit !== undefined) {
|
|
150
|
-
const n = query.limit;
|
|
151
|
-
if (typeof n !== 'number' || !Number.isInteger(n) || n < 1 || n > MAX_ROWS) {
|
|
152
|
-
return `limit must be a whole number from 1 to ${MAX_ROWS}`;
|
|
153
|
-
}
|
|
154
|
-
}
|
|
155
|
-
if (query.environment !== undefined && typeof query.environment !== 'string') {
|
|
156
|
-
return 'environment must be a string';
|
|
157
|
-
}
|
|
158
|
-
return null;
|
|
159
|
-
}
|
|
160
|
-
/**
|
|
161
|
-
* Answer a query from a cube's daily rows: keep the range and the filter,
|
|
162
|
-
* group, sum, sort, cut. Throws `invalid` for a query `validateQuery` refuses.
|
|
163
|
-
*/
|
|
164
|
-
export function runQuery(rows, q) {
|
|
165
|
-
const problem = validateQuery(q);
|
|
166
|
-
if (problem)
|
|
167
|
-
throw new AnalyticsError('invalid', problem);
|
|
168
|
-
const { metrics } = CUBES[q.cube];
|
|
169
|
-
const groupBy = [...(q.groupBy ?? [])];
|
|
170
|
-
const filter = Object.entries(q.filter ?? {}).map(([d, v]) => [d, Array.isArray(v) ? v : [v]]);
|
|
171
|
-
const groups = new Map();
|
|
172
|
-
for (const row of rows) {
|
|
173
|
-
const day = String(row.day);
|
|
174
|
-
if (day < q.from || day > q.to)
|
|
175
|
-
continue;
|
|
176
|
-
if (!filter.every(([d, values]) => values.includes(row[d] ?? null)))
|
|
177
|
-
continue;
|
|
178
|
-
const key = JSON.stringify(groupBy.map((d) => row[d] ?? null));
|
|
179
|
-
let out = groups.get(key);
|
|
180
|
-
if (!out) {
|
|
181
|
-
out = Object.fromEntries([
|
|
182
|
-
...groupBy.map((d) => [d, row[d] ?? null]),
|
|
183
|
-
...metrics.map((m) => [m, 0]),
|
|
184
|
-
]);
|
|
185
|
-
groups.set(key, out);
|
|
186
|
-
}
|
|
187
|
-
for (const m of metrics)
|
|
188
|
-
out[m] = out[m] + Number(row[m] ?? 0);
|
|
189
|
-
}
|
|
190
|
-
let out = [...groups.values()];
|
|
191
|
-
// Totals over an empty range are still one row of zeros, so a caller can read `rows[0].views`.
|
|
192
|
-
if (out.length === 0 && groupBy.length === 0) {
|
|
193
|
-
out = [Object.fromEntries(metrics.map((m) => [m, 0]))];
|
|
194
|
-
}
|
|
195
|
-
const order = q.order ?? groupBy[0];
|
|
196
|
-
if (order) {
|
|
197
|
-
const desc = order.startsWith('-');
|
|
198
|
-
const field = desc ? order.slice(1) : order;
|
|
199
|
-
out.sort((a, b) => compare(a[field] ?? null, b[field] ?? null, desc));
|
|
200
|
-
}
|
|
201
|
-
return out.slice(0, q.limit ?? MAX_ROWS);
|
|
202
|
-
}
|
|
203
|
-
/** Nulls last either way; numbers numerically, strings by code point (so days sort as days). */
|
|
204
|
-
function compare(a, b, desc) {
|
|
205
|
-
if (a === b)
|
|
206
|
-
return 0;
|
|
207
|
-
if (a === null)
|
|
208
|
-
return 1;
|
|
209
|
-
if (b === null)
|
|
210
|
-
return -1;
|
|
211
|
-
const c = typeof a === 'number' && typeof b === 'number' ? a - b : String(a) < String(b) ? -1 : 1;
|
|
212
|
-
return desc ? -c : c;
|
|
213
|
-
}
|