@knowledgefutures/sdk 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +24 -2
- package/dist/analytics/beacon.d.ts +25 -0
- package/dist/analytics/beacon.js +86 -0
- package/dist/analytics/binding.d.ts +24 -0
- package/dist/analytics/binding.js +46 -0
- package/dist/analytics/collector.d.ts +10 -0
- package/dist/analytics/collector.js +10 -0
- package/dist/analytics/cubes.d.ts +96 -0
- package/dist/analytics/cubes.js +213 -0
- package/dist/analytics/envelope.d.ts +135 -0
- package/dist/analytics/envelope.js +231 -0
- package/dist/analytics/errors.d.ts +24 -0
- package/dist/analytics/errors.js +20 -0
- package/dist/analytics/index.d.ts +38 -0
- package/dist/analytics/index.js +98 -0
- package/dist/analytics/memory.d.ts +35 -0
- package/dist/analytics/memory.js +66 -0
- package/dist/analytics/testing.d.ts +6 -0
- package/dist/analytics/testing.js +6 -0
- package/dist/analytics/types.d.ts +22 -0
- package/dist/analytics/types.js +1 -0
- package/dist/storage/binding.d.ts +76 -0
- package/dist/storage/binding.js +97 -0
- package/dist/storage/brokered.d.ts +21 -0
- package/dist/storage/brokered.js +94 -0
- package/dist/storage/conformance.d.ts +24 -0
- package/dist/storage/conformance.js +311 -0
- package/dist/storage/errors.d.ts +27 -0
- package/dist/storage/errors.js +21 -0
- package/dist/storage/index.d.ts +34 -0
- package/dist/storage/index.js +77 -0
- package/dist/storage/local.d.ts +26 -0
- package/dist/storage/local.js +57 -0
- package/dist/storage/s3.d.ts +22 -0
- package/dist/storage/s3.js +254 -0
- package/dist/storage/shared.d.ts +45 -0
- package/dist/storage/shared.js +167 -0
- package/dist/storage/sigv4.d.ts +50 -0
- package/dist/storage/sigv4.js +128 -0
- package/dist/storage/testing.d.ts +25 -0
- package/dist/storage/testing.js +242 -0
- package/dist/storage/types.d.ts +122 -0
- package/dist/storage/types.js +13 -0
- package/package.json +30 -3
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Knowledge Futures, Inc.
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,25 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @knowledgefutures/sdk
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Clients for the services a [Knowledge Futures Console](https://console.knowledgefutures.org)
|
|
4
|
+
project declares, with nothing to configure: the console writes each service's
|
|
5
|
+
binding into every environment that declares it, and `kf dev` writes the same
|
|
6
|
+
variables pointing at local stand-ins.
|
|
7
|
+
|
|
8
|
+
```sh
|
|
9
|
+
npm i @knowledgefutures/sdk
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
```ts
|
|
13
|
+
import { storage } from '@knowledgefutures/sdk/storage' // kf/storage@1
|
|
14
|
+
await storage.put('covers/1.jpg', bytes, { contentType: 'image/jpeg' })
|
|
15
|
+
|
|
16
|
+
// In tests, an in-memory store with the same interface:
|
|
17
|
+
import { memoryStorage } from '@knowledgefutures/sdk/storage/testing'
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
`createStorage(env)` builds a client from an explicit environment, such as a
|
|
21
|
+
Worker's `env` when `nodejs_compat` is off.
|
|
22
|
+
|
|
23
|
+
What a project declares and what the services give it:
|
|
24
|
+
[console.knowledgefutures.org/llms.txt](https://console.knowledgefutures.org/llms.txt).
|
|
25
|
+
Source: `packages/sdk/` in [knowledgefutures/kf-console](https://github.com/knowledgefutures/kf-console).
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The page script, served by the collector at `/v1/b.js`. Every project loads
|
|
3
|
+
* the same line:
|
|
4
|
+
*
|
|
5
|
+
* <script src="https://<collector>/v1/b.js" defer></script>
|
|
6
|
+
*
|
|
7
|
+
* and a project whose domains the console can't list (PubPub's custom
|
|
8
|
+
* domains) adds `data-site="<site id>"`. Ported from the mitp-oecs beacon;
|
|
9
|
+
* three changes: it posts to wherever it was loaded from, it sends
|
|
10
|
+
* `text/plain` (so a cross-origin `sendBeacon` needs no preflight and still
|
|
11
|
+
* carries `Origin`, which is how the collector attributes it), and it reads
|
|
12
|
+
* `kf:scope`.
|
|
13
|
+
*
|
|
14
|
+
* What a page declares, the beacon reads:
|
|
15
|
+
*
|
|
16
|
+
* <meta name="kf:subject" content="pub:za65x7nh">
|
|
17
|
+
* <meta name="kf:subject-uri" content="https://doi.org/10.21428/…">
|
|
18
|
+
* <meta name="kf:scope" content="<community id>">
|
|
19
|
+
*
|
|
20
|
+
* One view per page load; `data-kf-event="download:pdf"` on any element
|
|
21
|
+
* reports that event on click; `window.kf.track(kind, name, props)` for the
|
|
22
|
+
* rest. No cookies, no storage, nothing about the reader beyond the referrer.
|
|
23
|
+
* Every failure is swallowed: analytics must never break a page.
|
|
24
|
+
*/
|
|
25
|
+
export declare const BEACON_JS = "(() => {\n try {\n const script = document.currentScript;\n if (!script || !script.src) return;\n const endpoint = new URL('e', script.src);\n const site = script.getAttribute('data-site');\n if (site) endpoint.searchParams.set('site', site);\n const EP = endpoint.toString();\n const meta = (n) => {\n const el = document.querySelector('meta[name=\"' + n + '\"]');\n return (el && el.getAttribute('content')) || undefined;\n };\n let utm;\n new URLSearchParams(location.search).forEach((v, k) => {\n const key = k.indexOf('utm_') === 0 ? k.slice(4) : '';\n if (v && ['source', 'medium', 'campaign', 'term', 'content'].indexOf(key) >= 0) {\n (utm = utm || {})[key] = String(v);\n }\n });\n const strs = (o) => {\n if (!o || typeof o !== 'object') return undefined;\n const out = {};\n let n = 0;\n for (const k in o) if (o[k] !== undefined && o[k] !== null) { out[k] = String(o[k]); n++; }\n return n ? out : undefined;\n };\n const send = (kind, name, props) => {\n try {\n const body = {\n kind: kind,\n subject: meta('kf:subject'),\n subjectUri: meta('kf:subject-uri'),\n scope: meta('kf:scope'),\n path: location.pathname,\n referrer: document.referrer || undefined,\n name: (typeof name === 'string' && name) || undefined,\n props: strs(props),\n utm: utm,\n };\n for (const k in body) if (body[k] === undefined) delete body[k];\n const json = JSON.stringify(body);\n if (navigator.sendBeacon && navigator.sendBeacon(EP, json)) return;\n fetch(EP, { method: 'POST', mode: 'no-cors', keepalive: true, credentials: 'omit',\n headers: { 'content-type': 'text/plain' }, body: json });\n } catch (e) {}\n };\n window.kf = { track: send };\n const view = () => send('view');\n if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', view, { once: true });\n else view();\n document.addEventListener('click', (e) => {\n try {\n const el = e.target && e.target.closest && e.target.closest('[data-kf-event]');\n if (!el) return;\n const parts = (el.getAttribute('data-kf-event') || '').split(':');\n const kind = parts.shift();\n if (kind) send(kind, parts.join(':'));\n } catch (err) {}\n }, true);\n } catch (e) {}\n})();\n";
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The page script, served by the collector at `/v1/b.js`. Every project loads
|
|
3
|
+
* the same line:
|
|
4
|
+
*
|
|
5
|
+
* <script src="https://<collector>/v1/b.js" defer></script>
|
|
6
|
+
*
|
|
7
|
+
* and a project whose domains the console can't list (PubPub's custom
|
|
8
|
+
* domains) adds `data-site="<site id>"`. Ported from the mitp-oecs beacon;
|
|
9
|
+
* three changes: it posts to wherever it was loaded from, it sends
|
|
10
|
+
* `text/plain` (so a cross-origin `sendBeacon` needs no preflight and still
|
|
11
|
+
* carries `Origin`, which is how the collector attributes it), and it reads
|
|
12
|
+
* `kf:scope`.
|
|
13
|
+
*
|
|
14
|
+
* What a page declares, the beacon reads:
|
|
15
|
+
*
|
|
16
|
+
* <meta name="kf:subject" content="pub:za65x7nh">
|
|
17
|
+
* <meta name="kf:subject-uri" content="https://doi.org/10.21428/…">
|
|
18
|
+
* <meta name="kf:scope" content="<community id>">
|
|
19
|
+
*
|
|
20
|
+
* One view per page load; `data-kf-event="download:pdf"` on any element
|
|
21
|
+
* reports that event on click; `window.kf.track(kind, name, props)` for the
|
|
22
|
+
* rest. No cookies, no storage, nothing about the reader beyond the referrer.
|
|
23
|
+
* Every failure is swallowed: analytics must never break a page.
|
|
24
|
+
*/
|
|
25
|
+
export const BEACON_JS = `(() => {
|
|
26
|
+
try {
|
|
27
|
+
const script = document.currentScript;
|
|
28
|
+
if (!script || !script.src) return;
|
|
29
|
+
const endpoint = new URL('e', script.src);
|
|
30
|
+
const site = script.getAttribute('data-site');
|
|
31
|
+
if (site) endpoint.searchParams.set('site', site);
|
|
32
|
+
const EP = endpoint.toString();
|
|
33
|
+
const meta = (n) => {
|
|
34
|
+
const el = document.querySelector('meta[name="' + n + '"]');
|
|
35
|
+
return (el && el.getAttribute('content')) || undefined;
|
|
36
|
+
};
|
|
37
|
+
let utm;
|
|
38
|
+
new URLSearchParams(location.search).forEach((v, k) => {
|
|
39
|
+
const key = k.indexOf('utm_') === 0 ? k.slice(4) : '';
|
|
40
|
+
if (v && ['source', 'medium', 'campaign', 'term', 'content'].indexOf(key) >= 0) {
|
|
41
|
+
(utm = utm || {})[key] = String(v);
|
|
42
|
+
}
|
|
43
|
+
});
|
|
44
|
+
const strs = (o) => {
|
|
45
|
+
if (!o || typeof o !== 'object') return undefined;
|
|
46
|
+
const out = {};
|
|
47
|
+
let n = 0;
|
|
48
|
+
for (const k in o) if (o[k] !== undefined && o[k] !== null) { out[k] = String(o[k]); n++; }
|
|
49
|
+
return n ? out : undefined;
|
|
50
|
+
};
|
|
51
|
+
const send = (kind, name, props) => {
|
|
52
|
+
try {
|
|
53
|
+
const body = {
|
|
54
|
+
kind: kind,
|
|
55
|
+
subject: meta('kf:subject'),
|
|
56
|
+
subjectUri: meta('kf:subject-uri'),
|
|
57
|
+
scope: meta('kf:scope'),
|
|
58
|
+
path: location.pathname,
|
|
59
|
+
referrer: document.referrer || undefined,
|
|
60
|
+
name: (typeof name === 'string' && name) || undefined,
|
|
61
|
+
props: strs(props),
|
|
62
|
+
utm: utm,
|
|
63
|
+
};
|
|
64
|
+
for (const k in body) if (body[k] === undefined) delete body[k];
|
|
65
|
+
const json = JSON.stringify(body);
|
|
66
|
+
if (navigator.sendBeacon && navigator.sendBeacon(EP, json)) return;
|
|
67
|
+
fetch(EP, { method: 'POST', mode: 'no-cors', keepalive: true, credentials: 'omit',
|
|
68
|
+
headers: { 'content-type': 'text/plain' }, body: json });
|
|
69
|
+
} catch (e) {}
|
|
70
|
+
};
|
|
71
|
+
window.kf = { track: send };
|
|
72
|
+
const view = () => send('view');
|
|
73
|
+
if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', view, { once: true });
|
|
74
|
+
else view();
|
|
75
|
+
document.addEventListener('click', (e) => {
|
|
76
|
+
try {
|
|
77
|
+
const el = e.target && e.target.closest && e.target.closest('[data-kf-event]');
|
|
78
|
+
if (!el) return;
|
|
79
|
+
const parts = (el.getAttribute('data-kf-event') || '').split(':');
|
|
80
|
+
const kind = parts.shift();
|
|
81
|
+
if (kind) send(kind, parts.join(':'));
|
|
82
|
+
} catch (err) {}
|
|
83
|
+
}, true);
|
|
84
|
+
} catch (e) {}
|
|
85
|
+
})();
|
|
86
|
+
`;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `KF_ANALYTICS_URL` — the whole contract between the console, which
|
|
3
|
+
* registers a site with the collector and writes the binding into a
|
|
4
|
+
* project's environment, and the SDK, which reads it. Both import this file,
|
|
5
|
+
* so the format has exactly one definition.
|
|
6
|
+
*
|
|
7
|
+
* https://<site>:<secret>@<collector host>[/<base path>]
|
|
8
|
+
*
|
|
9
|
+
* `http` only for a local collector. The site id is public (a keyed beacon
|
|
10
|
+
* carries it); the secret signs server events and queries, and never reaches
|
|
11
|
+
* a browser. `log:` is `kf dev`'s binding: events are printed, nothing is
|
|
12
|
+
* sent.
|
|
13
|
+
*/
|
|
14
|
+
export interface AnalyticsBinding {
|
|
15
|
+
/** The collector's origin and base path, no trailing slash. */
|
|
16
|
+
endpoint: string;
|
|
17
|
+
site: string;
|
|
18
|
+
secret: string;
|
|
19
|
+
}
|
|
20
|
+
export declare const ANALYTICS_ENV = "KF_ANALYTICS_URL";
|
|
21
|
+
export declare const LOG_BINDING = "log:";
|
|
22
|
+
export declare function encodeAnalyticsBinding(b: AnalyticsBinding): string;
|
|
23
|
+
/** The binding, or `'log'` for `kf dev`'s print-only binding. */
|
|
24
|
+
export declare function decodeAnalyticsBinding(value: string): AnalyticsBinding | 'log';
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { AnalyticsError } from './errors.js';
|
|
2
|
+
export const ANALYTICS_ENV = 'KF_ANALYTICS_URL';
|
|
3
|
+
export const LOG_BINDING = 'log:';
|
|
4
|
+
const SITE = /^[A-Za-z0-9_-]{1,64}$/;
|
|
5
|
+
export function encodeAnalyticsBinding(b) {
|
|
6
|
+
const endpoint = new URL(b.endpoint);
|
|
7
|
+
if (endpoint.protocol !== 'https:' && endpoint.protocol !== 'http:') {
|
|
8
|
+
throw new AnalyticsError('invalid', `endpoint must be http(s): ${b.endpoint}`);
|
|
9
|
+
}
|
|
10
|
+
if (!SITE.test(b.site))
|
|
11
|
+
throw new AnalyticsError('invalid', `not a site id: ${b.site}`);
|
|
12
|
+
if (!b.secret)
|
|
13
|
+
throw new AnalyticsError('invalid', 'the secret is empty');
|
|
14
|
+
const path = endpoint.pathname.replace(/\/+$/, '');
|
|
15
|
+
return `${endpoint.protocol}//${b.site}:${encodeURIComponent(b.secret)}@${endpoint.host}${path}`;
|
|
16
|
+
}
|
|
17
|
+
/** The binding, or `'log'` for `kf dev`'s print-only binding. */
|
|
18
|
+
export function decodeAnalyticsBinding(value) {
|
|
19
|
+
const v = value.trim();
|
|
20
|
+
if (v === LOG_BINDING)
|
|
21
|
+
return 'log';
|
|
22
|
+
let url;
|
|
23
|
+
try {
|
|
24
|
+
url = new URL(v);
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
throw bad('not a URL');
|
|
28
|
+
}
|
|
29
|
+
if (url.protocol !== 'https:' && url.protocol !== 'http:') {
|
|
30
|
+
throw bad(`the scheme is https (or http for a local collector), or the value is "${LOG_BINDING}"`);
|
|
31
|
+
}
|
|
32
|
+
const site = decodeURIComponent(url.username);
|
|
33
|
+
if (!SITE.test(site))
|
|
34
|
+
throw bad('the user part is the site id');
|
|
35
|
+
if (!url.password)
|
|
36
|
+
throw bad('the secret is missing');
|
|
37
|
+
return {
|
|
38
|
+
endpoint: `${url.protocol}//${url.host}${url.pathname.replace(/\/+$/, '')}`,
|
|
39
|
+
site,
|
|
40
|
+
secret: decodeURIComponent(url.password),
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
function bad(why) {
|
|
44
|
+
// Never echo the value: it carries the secret.
|
|
45
|
+
return new AnalyticsError('not_configured', `${ANALYTICS_ENV} is malformed: ${why}`);
|
|
46
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@knowledgefutures/sdk/analytics/collector` — what the `kf/analytics`
|
|
3
|
+
* collector imports: the event format, the query semantics and the page
|
|
4
|
+
* script. The SDK's client half uses the same files, which is the point:
|
|
5
|
+
* a hint the SDK accepts is a hint the collector accepts, and a query answers
|
|
6
|
+
* the same from the memory double as from the service.
|
|
7
|
+
*/
|
|
8
|
+
export { buildRow, device, FIELDS, KINDS, LIMITS, SCHEMA, UTM_KEYS, validateHint, type Device, type EventRow, type Hint, type Kind, type RowContext, type ServerHint, type Source, } from './envelope.js';
|
|
9
|
+
export { CUBES, cubeRows, MAX_ROWS, runQuery, validateQuery, type Cube, type CubeRow, type Query, type QueryResult, } from './cubes.js';
|
|
10
|
+
export { BEACON_JS } from './beacon.js';
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@knowledgefutures/sdk/analytics/collector` — what the `kf/analytics`
|
|
3
|
+
* collector imports: the event format, the query semantics and the page
|
|
4
|
+
* script. The SDK's client half uses the same files, which is the point:
|
|
5
|
+
* a hint the SDK accepts is a hint the collector accepts, and a query answers
|
|
6
|
+
* the same from the memory double as from the service.
|
|
7
|
+
*/
|
|
8
|
+
export { buildRow, device, FIELDS, KINDS, LIMITS, SCHEMA, UTM_KEYS, validateHint, } from './envelope.js';
|
|
9
|
+
export { CUBES, cubeRows, MAX_ROWS, runQuery, validateQuery, } from './cubes.js';
|
|
10
|
+
export { BEACON_JS } from './beacon.js';
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import type { EventRow } from './envelope.js';
|
|
2
|
+
/**
|
|
3
|
+
* The cubes: one site's events, collapsed into daily rows the compactor
|
|
4
|
+
* writes and every query reads. They are OECS's eleven summary files
|
|
5
|
+
* generalized — `pubs` and `collections` became `subjects`, keyed by
|
|
6
|
+
* `subject_type`, and every cube gains `scope`.
|
|
7
|
+
*
|
|
8
|
+
* Two rules hold everywhere:
|
|
9
|
+
*
|
|
10
|
+
* - **Bots are gone** (`device = 'bot'`) before anything is counted.
|
|
11
|
+
* - **Every metric is a count, so it sums across days.** A query over any
|
|
12
|
+
* range is a sum of daily rows, and exact. A metric that can't be summed —
|
|
13
|
+
* distinct subjects in a month, unique people — can never be a column.
|
|
14
|
+
*
|
|
15
|
+
* `views` are rows with `kind = 'view'`; `visits` are views with
|
|
16
|
+
* `is_unique` (the referrer heuristic: visits, never "unique visitors").
|
|
17
|
+
*
|
|
18
|
+
* `cubeRows` is the reference implementation, used by the memory double.
|
|
19
|
+
* The compactor computes the same rows in SQL, and the conformance suite
|
|
20
|
+
* holds the two to each other.
|
|
21
|
+
*/
|
|
22
|
+
export declare const CUBES: {
|
|
23
|
+
readonly days: {
|
|
24
|
+
readonly dims: readonly ["day", "scope"];
|
|
25
|
+
readonly metrics: readonly ["views", "visits", "downloads", "searches", "custom"];
|
|
26
|
+
};
|
|
27
|
+
readonly subjects: {
|
|
28
|
+
readonly dims: readonly ["day", "scope", "subject_type", "subject_id"];
|
|
29
|
+
readonly metrics: readonly ["views", "visits", "downloads"];
|
|
30
|
+
};
|
|
31
|
+
readonly pages: {
|
|
32
|
+
readonly dims: readonly ["day", "scope", "path"];
|
|
33
|
+
readonly metrics: readonly ["views", "visits"];
|
|
34
|
+
};
|
|
35
|
+
readonly referrers: {
|
|
36
|
+
readonly dims: readonly ["day", "scope", "referrer_host"];
|
|
37
|
+
readonly metrics: readonly ["views", "visits"];
|
|
38
|
+
};
|
|
39
|
+
readonly campaigns: {
|
|
40
|
+
readonly dims: readonly ["day", "scope", "utm_source", "utm_medium", "utm_campaign"];
|
|
41
|
+
readonly metrics: readonly ["views", "visits"];
|
|
42
|
+
};
|
|
43
|
+
readonly devices: {
|
|
44
|
+
readonly dims: readonly ["day", "scope", "device"];
|
|
45
|
+
readonly metrics: readonly ["views", "visits"];
|
|
46
|
+
};
|
|
47
|
+
readonly countries: {
|
|
48
|
+
readonly dims: readonly ["day", "scope", "country"];
|
|
49
|
+
readonly metrics: readonly ["views", "visits"];
|
|
50
|
+
};
|
|
51
|
+
readonly downloads: {
|
|
52
|
+
readonly dims: readonly ["day", "scope", "subject_type", "subject_id", "name"];
|
|
53
|
+
readonly metrics: readonly ["downloads"];
|
|
54
|
+
};
|
|
55
|
+
readonly searches: {
|
|
56
|
+
readonly dims: readonly ["day", "scope", "name"];
|
|
57
|
+
readonly metrics: readonly ["searches"];
|
|
58
|
+
};
|
|
59
|
+
};
|
|
60
|
+
export type Cube = keyof typeof CUBES;
|
|
61
|
+
export type Dim<C extends Cube = Cube> = (typeof CUBES)[C]['dims'][number];
|
|
62
|
+
export type Metric<C extends Cube = Cube> = (typeof CUBES)[C]['metrics'][number];
|
|
63
|
+
/** A cube row: its dimensions (null where the event had none) and its counts. */
|
|
64
|
+
export type CubeRow = Record<string, string | number | null>;
|
|
65
|
+
/** One cube's daily rows from raw events, sorted by day. The compactor's contract, in code. */
|
|
66
|
+
export declare function cubeRows(cube: Cube, events: readonly EventRow[]): CubeRow[];
|
|
67
|
+
export interface Query<C extends Cube = Cube> {
|
|
68
|
+
cube: C;
|
|
69
|
+
/** First closed UTC day, inclusive, `YYYY-MM-DD`. */
|
|
70
|
+
from: string;
|
|
71
|
+
/** Last closed UTC day, inclusive. */
|
|
72
|
+
to: string;
|
|
73
|
+
/** Equality on a dimension; an array is "any of". `null` matches rows without the dimension. */
|
|
74
|
+
filter?: Partial<Record<Dim<C>, string | null | ReadonlyArray<string | null>>>;
|
|
75
|
+
/** The dimensions to keep. Omitted or `[]`: one row of totals. */
|
|
76
|
+
groupBy?: ReadonlyArray<Dim<C>>;
|
|
77
|
+
/** A metric or dimension, `-` first for descending. Default: the group's dimensions, ascending. */
|
|
78
|
+
order?: string;
|
|
79
|
+
/** At most `MAX_ROWS`. */
|
|
80
|
+
limit?: number;
|
|
81
|
+
/** Another of the project's environments. Default: the caller's own. */
|
|
82
|
+
environment?: string;
|
|
83
|
+
}
|
|
84
|
+
export interface QueryResult {
|
|
85
|
+
rows: CubeRow[];
|
|
86
|
+
/** The last day the cubes cover, or null when the site has no data yet. Today is never in them. */
|
|
87
|
+
through: string | null;
|
|
88
|
+
}
|
|
89
|
+
export declare const MAX_ROWS = 10000;
|
|
90
|
+
/** Why a query is refused, or null. The collector runs this too, so both answer alike. */
|
|
91
|
+
export declare function validateQuery(q: unknown): string | null;
|
|
92
|
+
/**
|
|
93
|
+
* Answer a query from a cube's daily rows: keep the range and the filter,
|
|
94
|
+
* group, sum, sort, cut. Throws `invalid` for a query `validateQuery` refuses.
|
|
95
|
+
*/
|
|
96
|
+
export declare function runQuery(rows: readonly CubeRow[], q: Query): CubeRow[];
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
import { AnalyticsError } from './errors.js';
|
|
2
|
+
/**
|
|
3
|
+
* The cubes: one site's events, collapsed into daily rows the compactor
|
|
4
|
+
* writes and every query reads. They are OECS's eleven summary files
|
|
5
|
+
* generalized — `pubs` and `collections` became `subjects`, keyed by
|
|
6
|
+
* `subject_type`, and every cube gains `scope`.
|
|
7
|
+
*
|
|
8
|
+
* Two rules hold everywhere:
|
|
9
|
+
*
|
|
10
|
+
* - **Bots are gone** (`device = 'bot'`) before anything is counted.
|
|
11
|
+
* - **Every metric is a count, so it sums across days.** A query over any
|
|
12
|
+
* range is a sum of daily rows, and exact. A metric that can't be summed —
|
|
13
|
+
* distinct subjects in a month, unique people — can never be a column.
|
|
14
|
+
*
|
|
15
|
+
* `views` are rows with `kind = 'view'`; `visits` are views with
|
|
16
|
+
* `is_unique` (the referrer heuristic: visits, never "unique visitors").
|
|
17
|
+
*
|
|
18
|
+
* `cubeRows` is the reference implementation, used by the memory double.
|
|
19
|
+
* The compactor computes the same rows in SQL, and the conformance suite
|
|
20
|
+
* holds the two to each other.
|
|
21
|
+
*/
|
|
22
|
+
export const CUBES = {
|
|
23
|
+
days: { dims: ['day', 'scope'], metrics: ['views', 'visits', 'downloads', 'searches', 'custom'] },
|
|
24
|
+
subjects: {
|
|
25
|
+
dims: ['day', 'scope', 'subject_type', 'subject_id'],
|
|
26
|
+
metrics: ['views', 'visits', 'downloads'],
|
|
27
|
+
},
|
|
28
|
+
pages: { dims: ['day', 'scope', 'path'], metrics: ['views', 'visits'] },
|
|
29
|
+
referrers: { dims: ['day', 'scope', 'referrer_host'], metrics: ['views', 'visits'] },
|
|
30
|
+
campaigns: {
|
|
31
|
+
dims: ['day', 'scope', 'utm_source', 'utm_medium', 'utm_campaign'],
|
|
32
|
+
metrics: ['views', 'visits'],
|
|
33
|
+
},
|
|
34
|
+
devices: { dims: ['day', 'scope', 'device'], metrics: ['views', 'visits'] },
|
|
35
|
+
countries: { dims: ['day', 'scope', 'country'], metrics: ['views', 'visits'] },
|
|
36
|
+
downloads: {
|
|
37
|
+
dims: ['day', 'scope', 'subject_type', 'subject_id', 'name'],
|
|
38
|
+
metrics: ['downloads'],
|
|
39
|
+
},
|
|
40
|
+
searches: { dims: ['day', 'scope', 'name'], metrics: ['searches'] },
|
|
41
|
+
};
|
|
42
|
+
/** Which events a cube counts, beyond "not a bot". */
|
|
43
|
+
const KEEP = {
|
|
44
|
+
days: () => true,
|
|
45
|
+
subjects: (e) => e.subject_type !== null && (e.kind === 'view' || e.kind === 'download'),
|
|
46
|
+
pages: (e) => e.kind === 'view',
|
|
47
|
+
// External referrers only: a click from one page of the site to another is not a referral.
|
|
48
|
+
referrers: (e) => e.kind === 'view' && e.referrer_host !== null && e.referrer_host !== e.hostname,
|
|
49
|
+
campaigns: (e) => e.kind === 'view' &&
|
|
50
|
+
(e.utm_source !== null || e.utm_medium !== null || e.utm_campaign !== null),
|
|
51
|
+
devices: (e) => e.kind === 'view',
|
|
52
|
+
countries: (e) => e.kind === 'view',
|
|
53
|
+
downloads: (e) => e.kind === 'download',
|
|
54
|
+
searches: (e) => e.kind === 'search',
|
|
55
|
+
};
|
|
56
|
+
function counts(e) {
|
|
57
|
+
return {
|
|
58
|
+
views: e.kind === 'view' ? 1 : 0,
|
|
59
|
+
visits: e.kind === 'view' && e.is_unique ? 1 : 0,
|
|
60
|
+
downloads: e.kind === 'download' ? 1 : 0,
|
|
61
|
+
searches: e.kind === 'search' ? 1 : 0,
|
|
62
|
+
custom: e.kind === 'custom' ? 1 : 0,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
/** One cube's daily rows from raw events, sorted by day. The compactor's contract, in code. */
|
|
66
|
+
export function cubeRows(cube, events) {
|
|
67
|
+
const { dims, metrics } = CUBES[cube];
|
|
68
|
+
const keep = KEEP[cube];
|
|
69
|
+
const rows = new Map();
|
|
70
|
+
for (const e of events) {
|
|
71
|
+
if (e.device === 'bot' || !keep(e))
|
|
72
|
+
continue;
|
|
73
|
+
const key = JSON.stringify(dims.map((d) => e[d]));
|
|
74
|
+
let row = rows.get(key);
|
|
75
|
+
if (!row) {
|
|
76
|
+
row = Object.fromEntries([
|
|
77
|
+
...dims.map((d) => [d, e[d]]),
|
|
78
|
+
...metrics.map((m) => [m, 0]),
|
|
79
|
+
]);
|
|
80
|
+
rows.set(key, row);
|
|
81
|
+
}
|
|
82
|
+
const c = counts(e);
|
|
83
|
+
for (const m of metrics)
|
|
84
|
+
row[m] = row[m] + c[m];
|
|
85
|
+
}
|
|
86
|
+
return [...rows.values()].sort((a, b) => String(a.day).localeCompare(String(b.day)));
|
|
87
|
+
}
|
|
88
|
+
export const MAX_ROWS = 10_000;
|
|
89
|
+
const DAY = /^\d{4}-\d{2}-\d{2}$/;
|
|
90
|
+
/** Why a query is refused, or null. The collector runs this too, so both answer alike. */
|
|
91
|
+
export function validateQuery(q) {
|
|
92
|
+
if (!q || typeof q !== 'object' || Array.isArray(q))
|
|
93
|
+
return 'query must be an object';
|
|
94
|
+
const query = q;
|
|
95
|
+
const known = new Set([
|
|
96
|
+
'cube',
|
|
97
|
+
'from',
|
|
98
|
+
'to',
|
|
99
|
+
'filter',
|
|
100
|
+
'groupBy',
|
|
101
|
+
'order',
|
|
102
|
+
'limit',
|
|
103
|
+
'environment',
|
|
104
|
+
]);
|
|
105
|
+
for (const key of Object.keys(query))
|
|
106
|
+
if (!known.has(key))
|
|
107
|
+
return `unknown key "${key}"`;
|
|
108
|
+
if (typeof query.cube !== 'string' || !Object.hasOwn(CUBES, query.cube)) {
|
|
109
|
+
return `cube must be one of ${Object.keys(CUBES).join(', ')}`;
|
|
110
|
+
}
|
|
111
|
+
const { dims, metrics } = CUBES[query.cube];
|
|
112
|
+
const isDim = (d) => dims.includes(d);
|
|
113
|
+
for (const key of ['from', 'to']) {
|
|
114
|
+
const v = query[key];
|
|
115
|
+
if (typeof v !== 'string' || !DAY.test(v) || Number.isNaN(Date.parse(v))) {
|
|
116
|
+
return `${key} must be a day, YYYY-MM-DD`;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
if (query.from > query.to)
|
|
120
|
+
return 'from is after to';
|
|
121
|
+
if (query.filter !== undefined) {
|
|
122
|
+
if (!query.filter || typeof query.filter !== 'object' || Array.isArray(query.filter)) {
|
|
123
|
+
return 'filter must be an object';
|
|
124
|
+
}
|
|
125
|
+
for (const [d, v] of Object.entries(query.filter)) {
|
|
126
|
+
if (!isDim(d))
|
|
127
|
+
return `${query.cube} has no dimension "${d}"; it has ${dims.join(', ')}`;
|
|
128
|
+
const values = Array.isArray(v) ? v : [v];
|
|
129
|
+
if (values.some((x) => x !== null && typeof x !== 'string')) {
|
|
130
|
+
return `filter.${d} must be a string, null, or a list of them`;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
if (query.groupBy !== undefined) {
|
|
135
|
+
if (!Array.isArray(query.groupBy))
|
|
136
|
+
return 'groupBy must be a list';
|
|
137
|
+
for (const d of query.groupBy) {
|
|
138
|
+
if (!isDim(d))
|
|
139
|
+
return `${query.cube} has no dimension "${String(d)}"; it has ${dims.join(', ')}`;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
if (query.order !== undefined) {
|
|
143
|
+
const field = typeof query.order === 'string' ? query.order.replace(/^-/, '') : undefined;
|
|
144
|
+
const grouped = query.groupBy ?? [];
|
|
145
|
+
if (!field || !(metrics.includes(field) || grouped.includes(field))) {
|
|
146
|
+
return 'order must be a metric or a grouped dimension, "-" first for descending';
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
if (query.limit !== undefined) {
|
|
150
|
+
const n = query.limit;
|
|
151
|
+
if (typeof n !== 'number' || !Number.isInteger(n) || n < 1 || n > MAX_ROWS) {
|
|
152
|
+
return `limit must be a whole number from 1 to ${MAX_ROWS}`;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
if (query.environment !== undefined && typeof query.environment !== 'string') {
|
|
156
|
+
return 'environment must be a string';
|
|
157
|
+
}
|
|
158
|
+
return null;
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Answer a query from a cube's daily rows: keep the range and the filter,
|
|
162
|
+
* group, sum, sort, cut. Throws `invalid` for a query `validateQuery` refuses.
|
|
163
|
+
*/
|
|
164
|
+
export function runQuery(rows, q) {
|
|
165
|
+
const problem = validateQuery(q);
|
|
166
|
+
if (problem)
|
|
167
|
+
throw new AnalyticsError('invalid', problem);
|
|
168
|
+
const { metrics } = CUBES[q.cube];
|
|
169
|
+
const groupBy = [...(q.groupBy ?? [])];
|
|
170
|
+
const filter = Object.entries(q.filter ?? {}).map(([d, v]) => [d, Array.isArray(v) ? v : [v]]);
|
|
171
|
+
const groups = new Map();
|
|
172
|
+
for (const row of rows) {
|
|
173
|
+
const day = String(row.day);
|
|
174
|
+
if (day < q.from || day > q.to)
|
|
175
|
+
continue;
|
|
176
|
+
if (!filter.every(([d, values]) => values.includes(row[d] ?? null)))
|
|
177
|
+
continue;
|
|
178
|
+
const key = JSON.stringify(groupBy.map((d) => row[d] ?? null));
|
|
179
|
+
let out = groups.get(key);
|
|
180
|
+
if (!out) {
|
|
181
|
+
out = Object.fromEntries([
|
|
182
|
+
...groupBy.map((d) => [d, row[d] ?? null]),
|
|
183
|
+
...metrics.map((m) => [m, 0]),
|
|
184
|
+
]);
|
|
185
|
+
groups.set(key, out);
|
|
186
|
+
}
|
|
187
|
+
for (const m of metrics)
|
|
188
|
+
out[m] = out[m] + Number(row[m] ?? 0);
|
|
189
|
+
}
|
|
190
|
+
let out = [...groups.values()];
|
|
191
|
+
// Totals over an empty range are still one row of zeros, so a caller can read `rows[0].views`.
|
|
192
|
+
if (out.length === 0 && groupBy.length === 0) {
|
|
193
|
+
out = [Object.fromEntries(metrics.map((m) => [m, 0]))];
|
|
194
|
+
}
|
|
195
|
+
const order = q.order ?? groupBy[0];
|
|
196
|
+
if (order) {
|
|
197
|
+
const desc = order.startsWith('-');
|
|
198
|
+
const field = desc ? order.slice(1) : order;
|
|
199
|
+
out.sort((a, b) => compare(a[field] ?? null, b[field] ?? null, desc));
|
|
200
|
+
}
|
|
201
|
+
return out.slice(0, q.limit ?? MAX_ROWS);
|
|
202
|
+
}
|
|
203
|
+
/** Nulls last either way; numbers numerically, strings by code point (so days sort as days). */
|
|
204
|
+
function compare(a, b, desc) {
|
|
205
|
+
if (a === b)
|
|
206
|
+
return 0;
|
|
207
|
+
if (a === null)
|
|
208
|
+
return 1;
|
|
209
|
+
if (b === null)
|
|
210
|
+
return -1;
|
|
211
|
+
const c = typeof a === 'number' && typeof b === 'number' ? a - b : String(a) < String(b) ? -1 : 1;
|
|
212
|
+
return desc ? -c : c;
|
|
213
|
+
}
|