@skene/flows-sdk 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +11 -0
- package/README.md +59 -2
- package/dist/index.d.ts +112 -0
- package/dist/index.js +313 -0
- package/kit/apply.sql +430 -0
- package/kit/rollback.sql +28 -0
- package/package.json +30 -4
package/LICENSE
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
Copyright (c) 2026 Skene Technologies. All rights reserved.
|
|
2
|
+
|
|
3
|
+
@skene/flows-sdk is proprietary software of Skene Technologies. It is not open source, and no
|
|
4
|
+
license to copy, modify, distribute, sublicense or create derivative works of it is granted by
|
|
5
|
+
its publication on npm or anywhere else. Use of the package is permitted only under a separate
|
|
6
|
+
written agreement with Skene Technologies, such as the terms under which you use Skene.
|
|
7
|
+
|
|
8
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
|
9
|
+
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
10
|
+
NONINFRINGEMENT. IN NO EVENT SHALL SKENE TECHNOLOGIES BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
11
|
+
LIABILITY ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR ITS USE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,60 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @skene/flows-sdk
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
The Skene Flows SDK sends product events straight into your own database. Skene then reads them
|
|
4
|
+
from there with a read-only role, so no event, payload or key reaches Skene.
|
|
5
|
+
|
|
6
|
+
1. Install the SDK: `npm install @skene/flows-sdk`.
|
|
7
|
+
2. Apply the Flows SDK kit in your database, as the role that will own schema `skene`. It ships in
|
|
8
|
+
this package as `kit/apply.sql` (after the install, `node_modules/@skene/flows-sdk/kit/apply.sql`),
|
|
9
|
+
with its rollback beside it as `kit/rollback.sql`. It creates schema `skene` with the events table
|
|
10
|
+
and the one collection function, `skene.collect_events`.
|
|
11
|
+
3. On Supabase, add `skene` to the Data API's exposed schemas.
|
|
12
|
+
4. Send events:
|
|
13
|
+
|
|
14
|
+
```ts
|
|
15
|
+
import { browserStores, createFlowsClient, sendOnPageHide, supabaseTransport } from '@skene/flows-sdk';
|
|
16
|
+
|
|
17
|
+
const flows = createFlowsClient({
|
|
18
|
+
transport: supabaseTransport({ url: 'https://<project-ref>.supabase.co', key: '<publishable key>' }),
|
|
19
|
+
...browserStores(),
|
|
20
|
+
});
|
|
21
|
+
sendOnPageHide(flows);
|
|
22
|
+
|
|
23
|
+
flows.track('visit');
|
|
24
|
+
flows.identify(user.id);
|
|
25
|
+
flows.track('signup', { plan: 'pro' });
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
How it behaves:
|
|
29
|
+
|
|
30
|
+
- Each event gets a UUID once, which is kept across retries, and a sequence number within its
|
|
31
|
+
installation. Each `createFlowsClient` is a new installation; only the unsent buffer is kept per
|
|
32
|
+
tab, so a reload resends it.
|
|
33
|
+
- An event leaves the buffer only when your database answers `accepted` or `duplicate`.
|
|
34
|
+
`conflict` and `invalid` are dropped and reported to `onDrop`. Anything else is retried with
|
|
35
|
+
backoff.
|
|
36
|
+
- `onReceipt` receives each event your database answered `accepted` or `duplicate`, with that
|
|
37
|
+
answer and the time the database first received the event. The answer comes only after your
|
|
38
|
+
database committed the event, and the event has already left the buffer, so a receipt you record
|
|
39
|
+
always names an event your database holds. A page-hide send, a failed or unanswered request and a
|
|
40
|
+
refused event get no receipt. A `duplicate` carries the first receive time. An error your callback
|
|
41
|
+
throws is ignored.
|
|
42
|
+
- A send on page hide reads no answer, so it never clears an event.
|
|
43
|
+
- `identify` changes later events only.
|
|
44
|
+
- When the buffer is full, a new event is dropped after it is numbered. Its loss shows as a gap.
|
|
45
|
+
A server installation's gap stops the interval from being sealed; a browser installation's gap is
|
|
46
|
+
reported in the package's manifest for you to judge.
|
|
47
|
+
- Your database takes an event only if it is timed at most 5 minutes ahead of its clock and at most
|
|
48
|
+
7 days behind it. An event kept unsent longer is answered `invalid`.
|
|
49
|
+
- `track` and `identify` refuse text holding a lone surrogate or a NUL, which PostgreSQL cannot
|
|
50
|
+
store. If your database still refuses a whole batch (HTTP 400), the SDK splits it until the one
|
|
51
|
+
bad event is alone and drops it as `invalid`.
|
|
52
|
+
- Events sent with the publishable key are recorded as sent from a browser, because anyone with
|
|
53
|
+
that key can send them. Send conversion-critical events from your server, with your own
|
|
54
|
+
transport or the secret key.
|
|
55
|
+
|
|
56
|
+
## License
|
|
57
|
+
|
|
58
|
+
Proprietary. Copyright (c) 2026 Skene Technologies, all rights reserved; see [LICENSE](LICENSE).
|
|
59
|
+
The package is public on npm so that you can install it, but it is not open source: it is for use
|
|
60
|
+
with Skene under your agreement with Skene Technologies.
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/** A property value the measurement can seal: short text, a safe integer or a boolean. */
|
|
2
|
+
export type PropertyValue = string | number | boolean;
|
|
3
|
+
/** One event as the collection function receives it. */
|
|
4
|
+
export type FlowsEvent = Readonly<{
|
|
5
|
+
event_id: string;
|
|
6
|
+
event_name: string;
|
|
7
|
+
occurred_at: string;
|
|
8
|
+
person_id: string | null;
|
|
9
|
+
anonymous_id: string | null;
|
|
10
|
+
installation_id: string;
|
|
11
|
+
sequence: number;
|
|
12
|
+
properties?: Readonly<Record<string, PropertyValue>>;
|
|
13
|
+
}>;
|
|
14
|
+
export type ReceiptStatus = 'accepted' | 'duplicate' | 'conflict' | 'invalid';
|
|
15
|
+
/** The database's answer for one event, after its transaction committed. */
|
|
16
|
+
export type Receipt = Readonly<{
|
|
17
|
+
event_id: string | null;
|
|
18
|
+
status: ReceiptStatus;
|
|
19
|
+
received_at: string | null;
|
|
20
|
+
}>;
|
|
21
|
+
/**
|
|
22
|
+
* Sends one batch and returns the database's receipts. It throws when there is no answer to trust,
|
|
23
|
+
* and throws a RefusedBatchError when the database refused the batch whole because of what it holds.
|
|
24
|
+
* With keepalive the request may outlive the page, and its answer is never read.
|
|
25
|
+
*/
|
|
26
|
+
export type Transport = (events: readonly FlowsEvent[], options: Readonly<{
|
|
27
|
+
keepalive: boolean;
|
|
28
|
+
}>) => Promise<readonly Receipt[]>;
|
|
29
|
+
/** Where the SDK keeps state between page loads. Every method may throw; the SDK then keeps memory only. */
|
|
30
|
+
export type Store = Readonly<{
|
|
31
|
+
get(key: string): string | null;
|
|
32
|
+
set(key: string, value: string): void;
|
|
33
|
+
}>;
|
|
34
|
+
export type FlowsOptions = Readonly<{
|
|
35
|
+
transport: Transport;
|
|
36
|
+
/** Kept per device: the anonymous id and the identified person. Default: memory. */
|
|
37
|
+
deviceStore?: Store;
|
|
38
|
+
/** Kept per tab or process: the unsent buffer, so a reload resends it. Default: memory. */
|
|
39
|
+
bufferStore?: Store;
|
|
40
|
+
/** Events kept while unsent, 1 to 10,000. Default 1,000. */
|
|
41
|
+
maxBuffer?: number;
|
|
42
|
+
/** Delay before the first retry, doubled per failure up to maxRetryMs. Defaults 1 s and 60 s. */
|
|
43
|
+
initialRetryMs?: number;
|
|
44
|
+
maxRetryMs?: number;
|
|
45
|
+
/** Delay between a track call and the send it schedules. Default 1 s. */
|
|
46
|
+
flushDelayMs?: number;
|
|
47
|
+
/** Called once per event the database refused, or the full buffer dropped. */
|
|
48
|
+
onDrop?: (event: FlowsEvent, reason: 'conflict' | 'invalid' | 'buffer_full') => void;
|
|
49
|
+
/**
|
|
50
|
+
* Called once per event the database acknowledged (accepted or duplicate), with its receipt, after
|
|
51
|
+
* the event has left the saved buffer. The answer exists only after the database committed the
|
|
52
|
+
* event, so this is never called for an event the database does not hold: not for a page-hide
|
|
53
|
+
* send, a failed or unanswered request, or a refused event. An error it throws is ignored.
|
|
54
|
+
*/
|
|
55
|
+
onReceipt?: (event: FlowsEvent, receipt: Receipt) => void;
|
|
56
|
+
/** Injected for tests; defaults are the platform's. */
|
|
57
|
+
now?: () => number;
|
|
58
|
+
uuid?: () => string;
|
|
59
|
+
random?: () => number;
|
|
60
|
+
schedule?: (run: () => void, delayMs: number) => unknown;
|
|
61
|
+
cancel?: (handle: unknown) => void;
|
|
62
|
+
}>;
|
|
63
|
+
export type FlushResult = Readonly<{
|
|
64
|
+
sent: number;
|
|
65
|
+
cleared: number;
|
|
66
|
+
dropped: number;
|
|
67
|
+
failed: boolean;
|
|
68
|
+
}>;
|
|
69
|
+
export type FlowsClient = Readonly<{
|
|
70
|
+
/** Numbers, buffers and schedules one event; returns it, or null when the buffer was full and it was dropped. */
|
|
71
|
+
track(eventName: string, properties?: Readonly<Record<string, PropertyValue>>): FlowsEvent | null;
|
|
72
|
+
/** Later events carry this person id instead of the anonymous id; earlier events are never rewritten. */
|
|
73
|
+
identify(personId: string): void;
|
|
74
|
+
/** Sends the oldest batch now and processes its receipts. One send runs at a time. */
|
|
75
|
+
flush(): Promise<FlushResult>;
|
|
76
|
+
/** Sends the oldest batch with keepalive and reads no answer, so nothing leaves the buffer. */
|
|
77
|
+
flushOnHide(): void;
|
|
78
|
+
/** The unsent events, oldest first. */
|
|
79
|
+
pending(): readonly FlowsEvent[];
|
|
80
|
+
readonly installationId: string;
|
|
81
|
+
}>;
|
|
82
|
+
/** The database refused a batch whole because of what it holds (HTTP 400), so resending it unchanged cannot help. */
|
|
83
|
+
export declare class RefusedBatchError extends Error {
|
|
84
|
+
constructor(message: string);
|
|
85
|
+
}
|
|
86
|
+
export declare function createFlowsClient(options: FlowsOptions): FlowsClient;
|
|
87
|
+
/** Receipts read from a collection answer, or an error when it is not one. */
|
|
88
|
+
export declare function readReceipts(body: unknown): readonly Receipt[];
|
|
89
|
+
/**
|
|
90
|
+
* The transport for a Supabase project (or any PostgREST): it calls skene.collect_events through the
|
|
91
|
+
* project's Data API with its key. In a browser use the public key: those events are recorded as sent
|
|
92
|
+
* from a browser. A 400 answer (the database could not read what the batch holds) throws a
|
|
93
|
+
* RefusedBatchError, so the batch is split; any other non-2xx answer or an unreadable body throws,
|
|
94
|
+
* so the batch is retried.
|
|
95
|
+
*/
|
|
96
|
+
export declare function supabaseTransport(config: Readonly<{
|
|
97
|
+
url: string;
|
|
98
|
+
key: string;
|
|
99
|
+
schema?: string;
|
|
100
|
+
fetch?: typeof fetch;
|
|
101
|
+
}>): Transport;
|
|
102
|
+
/** Browser storage: localStorage for the device's identities, sessionStorage for the tab's unsent buffer. */
|
|
103
|
+
export declare function browserStores(): Readonly<{
|
|
104
|
+
deviceStore: Store;
|
|
105
|
+
bufferStore: Store;
|
|
106
|
+
}>;
|
|
107
|
+
type PageDocument = EventTarget & Readonly<{
|
|
108
|
+
visibilityState: string;
|
|
109
|
+
}>;
|
|
110
|
+
/** Sends the buffer with keepalive whenever the page is hidden; returns the function that stops it. */
|
|
111
|
+
export declare function sendOnPageHide(client: FlowsClient, page?: PageDocument): () => void;
|
|
112
|
+
export {};
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
// Plan row L2-4 (LCH-03.sdk): the Skene Flows SDK. It numbers and buffers events on the device and
|
|
2
|
+
// writes them into the customer's own database through skene.collect_events, the function the Flows
|
|
3
|
+
// SDK kit creates there (kit/apply.sql in this package). No event, payload or key
|
|
4
|
+
// reaches Skene. The rule record is docs/adr/20261010-flows-sdk-route-contract.md.
|
|
5
|
+
//
|
|
6
|
+
// An event leaves the buffer only when the database answers accepted or duplicate for its id, or
|
|
7
|
+
// refuses it as conflict or invalid (a retry cannot change either answer). A failed or unanswered
|
|
8
|
+
// request keeps every event and retries with exponential backoff. A batch the database refuses
|
|
9
|
+
// whole (HTTP 400) is halved until the one event it cannot read is alone, and that event is dropped
|
|
10
|
+
// as invalid, so it never holds back the events behind it. A send on page hide observes no answer,
|
|
11
|
+
// so it never clears an event. Every event is numbered within its installation before it is
|
|
12
|
+
// buffered, so an event that is dropped because the buffer is full leaves a sequence gap. An
|
|
13
|
+
// installation is one client: its id is new on every createFlowsClient and lives only in memory, so
|
|
14
|
+
// two contexts (a duplicated tab, a reload, a second process) never number events in one series.
|
|
15
|
+
const BATCH = 100;
|
|
16
|
+
const MAX_TEXT = 200;
|
|
17
|
+
const MAX_KEY = 120;
|
|
18
|
+
const MAX_PROPERTIES = 20;
|
|
19
|
+
const KEYS = { anonymous: 'skene_flows_anonymous_id', person: 'skene_flows_person_id', buffer: 'skene_flows_buffer' };
|
|
20
|
+
/** The database refused a batch whole because of what it holds (HTTP 400), so resending it unchanged cannot help. */
|
|
21
|
+
export class RefusedBatchError extends Error {
|
|
22
|
+
constructor(message) {
|
|
23
|
+
super(message);
|
|
24
|
+
this.name = 'RefusedBatchError';
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Text PostgreSQL can store as JSON: no lone UTF-16 surrogate (a string cut through an emoji) and no
|
|
29
|
+
* NUL. Either makes the database refuse the whole request before the collection function runs.
|
|
30
|
+
*/
|
|
31
|
+
function storable(value) {
|
|
32
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
33
|
+
const unit = value.charCodeAt(index);
|
|
34
|
+
if (unit === 0)
|
|
35
|
+
return false;
|
|
36
|
+
if (unit >= 0xdc00 && unit <= 0xdfff)
|
|
37
|
+
return false;
|
|
38
|
+
if (unit >= 0xd800 && unit <= 0xdbff) {
|
|
39
|
+
const next = value.charCodeAt(index + 1);
|
|
40
|
+
if (!(next >= 0xdc00 && next <= 0xdfff))
|
|
41
|
+
return false;
|
|
42
|
+
index += 1;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return true;
|
|
46
|
+
}
|
|
47
|
+
/** The kit's text rule: 1 to max characters, nothing trim() removes at either end, no C0 control character. */
|
|
48
|
+
function textOk(value, max = MAX_TEXT) {
|
|
49
|
+
if (typeof value !== 'string' || value.length === 0 || value.length > max || value.trim() !== value)
|
|
50
|
+
return false;
|
|
51
|
+
for (let index = 0; index < value.length; index += 1)
|
|
52
|
+
if (value.charCodeAt(index) < 32)
|
|
53
|
+
return false;
|
|
54
|
+
return storable(value);
|
|
55
|
+
}
|
|
56
|
+
function propertiesOk(value) {
|
|
57
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value))
|
|
58
|
+
return false;
|
|
59
|
+
const entries = Object.entries(value);
|
|
60
|
+
return entries.length >= 1 && entries.length <= MAX_PROPERTIES && entries.every(([key, item]) => textOk(key, MAX_KEY)
|
|
61
|
+
&& (typeof item === 'boolean' || typeof item === 'string' && item.length <= MAX_TEXT && storable(item)
|
|
62
|
+
|| typeof item === 'number' && Number.isSafeInteger(item)));
|
|
63
|
+
}
|
|
64
|
+
/** Reads a store, treating a missing or throwing store as empty: the SDK then keeps memory only. */
|
|
65
|
+
function read(store, key) {
|
|
66
|
+
try {
|
|
67
|
+
return store?.get(key) ?? null;
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
return null;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
function write(store, key, value) {
|
|
74
|
+
try {
|
|
75
|
+
store?.set(key, value);
|
|
76
|
+
}
|
|
77
|
+
catch { /* memory only: the store refused */ }
|
|
78
|
+
}
|
|
79
|
+
function storedEvents(text) {
|
|
80
|
+
if (text === null)
|
|
81
|
+
return [];
|
|
82
|
+
try {
|
|
83
|
+
const parsed = JSON.parse(text);
|
|
84
|
+
return Array.isArray(parsed) ? parsed.filter((item) => typeof item === 'object' && item !== null && typeof item.event_id === 'string') : [];
|
|
85
|
+
}
|
|
86
|
+
catch {
|
|
87
|
+
return [];
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
export function createFlowsClient(options) {
|
|
91
|
+
const device = options.deviceStore;
|
|
92
|
+
const bufferStore = options.bufferStore;
|
|
93
|
+
const maxBuffer = Math.min(10_000, Math.max(1, Math.trunc(options.maxBuffer ?? 1000)));
|
|
94
|
+
const initialRetryMs = options.initialRetryMs ?? 1000;
|
|
95
|
+
const maxRetryMs = options.maxRetryMs ?? 60_000;
|
|
96
|
+
const flushDelayMs = options.flushDelayMs ?? 1000;
|
|
97
|
+
const now = options.now ?? Date.now;
|
|
98
|
+
const uuid = options.uuid ?? (() => globalThis.crypto.randomUUID());
|
|
99
|
+
const random = options.random ?? Math.random;
|
|
100
|
+
const schedule = options.schedule ?? ((run, delayMs) => setTimeout(run, delayMs));
|
|
101
|
+
const cancel = options.cancel ?? ((handle) => clearTimeout(handle));
|
|
102
|
+
const remembered = (store, key) => {
|
|
103
|
+
const found = read(store, key);
|
|
104
|
+
if (found !== null && found.length > 0)
|
|
105
|
+
return found;
|
|
106
|
+
const created = uuid();
|
|
107
|
+
write(store, key, created);
|
|
108
|
+
return created;
|
|
109
|
+
};
|
|
110
|
+
const anonymousId = remembered(device, KEYS.anonymous);
|
|
111
|
+
// Never stored: a copied store (a duplicated tab, a window.open that keeps its opener) would
|
|
112
|
+
// otherwise give two contexts one series, and the second to send a number would lose its event as
|
|
113
|
+
// a conflict that no gap shows. A restored buffer keeps its events' own installation and numbers.
|
|
114
|
+
const installationId = uuid();
|
|
115
|
+
let personId = read(device, KEYS.person);
|
|
116
|
+
let nextSequence = 1;
|
|
117
|
+
let buffer = storedEvents(read(bufferStore, KEYS.buffer)).slice(0, maxBuffer);
|
|
118
|
+
let failures = 0;
|
|
119
|
+
let timer = null;
|
|
120
|
+
let sending = null;
|
|
121
|
+
// Halved on each whole-batch refusal until the event the database cannot read is alone.
|
|
122
|
+
let batchLimit = BATCH;
|
|
123
|
+
const save = () => write(bufferStore, KEYS.buffer, JSON.stringify(buffer));
|
|
124
|
+
const plan = (delayMs) => {
|
|
125
|
+
timer = schedule(() => { timer = null; void client.flush(); }, delayMs);
|
|
126
|
+
};
|
|
127
|
+
const retryDelay = () => Math.round(random() * Math.min(maxRetryMs, initialRetryMs * 2 ** Math.min(failures - 1, 30)));
|
|
128
|
+
const send = async () => {
|
|
129
|
+
const batch = buffer.slice(0, batchLimit);
|
|
130
|
+
if (batch.length === 0)
|
|
131
|
+
return { sent: 0, cleared: 0, dropped: 0, failed: false };
|
|
132
|
+
let receipts;
|
|
133
|
+
try {
|
|
134
|
+
receipts = await options.transport(batch, { keepalive: false });
|
|
135
|
+
}
|
|
136
|
+
catch (error) {
|
|
137
|
+
if (error instanceof RefusedBatchError) {
|
|
138
|
+
const [only] = batch;
|
|
139
|
+
if (batch.length === 1 && only !== undefined) {
|
|
140
|
+
buffer = buffer.filter(event => event !== only);
|
|
141
|
+
save();
|
|
142
|
+
batchLimit = BATCH;
|
|
143
|
+
options.onDrop?.(only, 'invalid');
|
|
144
|
+
if (buffer.length > 0)
|
|
145
|
+
plan(0);
|
|
146
|
+
return { sent: 1, cleared: 0, dropped: 1, failed: false };
|
|
147
|
+
}
|
|
148
|
+
batchLimit = Math.ceil(batch.length / 2);
|
|
149
|
+
plan(0);
|
|
150
|
+
return { sent: batch.length, cleared: 0, dropped: 0, failed: true };
|
|
151
|
+
}
|
|
152
|
+
failures += 1;
|
|
153
|
+
plan(retryDelay());
|
|
154
|
+
return { sent: batch.length, cleared: 0, dropped: 0, failed: true };
|
|
155
|
+
}
|
|
156
|
+
const answers = new Map();
|
|
157
|
+
// An answer that is not a list answers no event, so every event in the batch stays for a retry.
|
|
158
|
+
for (const receipt of Array.isArray(receipts) ? receipts : []) {
|
|
159
|
+
if (typeof receipt?.event_id === 'string')
|
|
160
|
+
answers.set(receipt.event_id, receipt);
|
|
161
|
+
}
|
|
162
|
+
const sentIds = new Set(batch.map(event => event.event_id));
|
|
163
|
+
const acknowledged = [];
|
|
164
|
+
let dropped = 0;
|
|
165
|
+
buffer = buffer.filter(event => {
|
|
166
|
+
if (!sentIds.has(event.event_id))
|
|
167
|
+
return true;
|
|
168
|
+
const receipt = answers.get(event.event_id);
|
|
169
|
+
if (receipt?.status === 'accepted' || receipt?.status === 'duplicate') {
|
|
170
|
+
acknowledged.push([event, receipt]);
|
|
171
|
+
return false;
|
|
172
|
+
}
|
|
173
|
+
if (receipt?.status === 'conflict' || receipt?.status === 'invalid') {
|
|
174
|
+
dropped += 1;
|
|
175
|
+
options.onDrop?.(event, receipt.status);
|
|
176
|
+
return false;
|
|
177
|
+
}
|
|
178
|
+
return true;
|
|
179
|
+
});
|
|
180
|
+
save();
|
|
181
|
+
const cleared = acknowledged.length;
|
|
182
|
+
// Plan row L2-8B (LCH-08.queues): a receipt is handed on only after the database answered, which
|
|
183
|
+
// PostgREST does only once the transaction committed, and only once its event has left the saved
|
|
184
|
+
// buffer. A throwing callback is the customer's error to handle: it is ignored here, so it can
|
|
185
|
+
// neither put the event back nor stop the events behind it from being sent.
|
|
186
|
+
for (const [event, receipt] of acknowledged) {
|
|
187
|
+
try {
|
|
188
|
+
options.onReceipt?.(event, receipt);
|
|
189
|
+
}
|
|
190
|
+
catch { /* see above */ }
|
|
191
|
+
}
|
|
192
|
+
const unanswered = batch.length - cleared - dropped;
|
|
193
|
+
// Backoff resets only when every event sent was answered; an unanswered event keeps doubling it.
|
|
194
|
+
if (unanswered > 0) {
|
|
195
|
+
failures += 1;
|
|
196
|
+
plan(retryDelay());
|
|
197
|
+
}
|
|
198
|
+
else {
|
|
199
|
+
failures = 0;
|
|
200
|
+
batchLimit = BATCH;
|
|
201
|
+
if (buffer.length > 0)
|
|
202
|
+
plan(0);
|
|
203
|
+
}
|
|
204
|
+
return { sent: batch.length, cleared, dropped, failed: unanswered > 0 };
|
|
205
|
+
};
|
|
206
|
+
const client = {
|
|
207
|
+
installationId,
|
|
208
|
+
track(eventName, properties) {
|
|
209
|
+
if (!textOk(eventName))
|
|
210
|
+
throw new TypeError('An event name is 1 to 200 characters, trimmed, without control characters');
|
|
211
|
+
if (properties !== undefined && !propertiesOk(properties)) {
|
|
212
|
+
throw new TypeError('Properties are 1 to 20 keys of at most 120 characters, each short text, a safe integer or a boolean');
|
|
213
|
+
}
|
|
214
|
+
const event = {
|
|
215
|
+
event_id: uuid(), event_name: eventName, occurred_at: new Date(now()).toISOString(),
|
|
216
|
+
person_id: personId, anonymous_id: personId === null ? anonymousId : null,
|
|
217
|
+
installation_id: installationId, sequence: nextSequence,
|
|
218
|
+
...(properties === undefined ? {} : { properties: { ...properties } }),
|
|
219
|
+
};
|
|
220
|
+
nextSequence += 1;
|
|
221
|
+
if (buffer.length >= maxBuffer) {
|
|
222
|
+
options.onDrop?.(event, 'buffer_full');
|
|
223
|
+
return null;
|
|
224
|
+
}
|
|
225
|
+
buffer.push(event);
|
|
226
|
+
save();
|
|
227
|
+
if (sending === null && timer === null)
|
|
228
|
+
plan(flushDelayMs);
|
|
229
|
+
return event;
|
|
230
|
+
},
|
|
231
|
+
identify(id) {
|
|
232
|
+
if (!textOk(id))
|
|
233
|
+
throw new TypeError('A person id is 1 to 200 characters, trimmed, without control characters');
|
|
234
|
+
personId = id;
|
|
235
|
+
write(device, KEYS.person, id);
|
|
236
|
+
},
|
|
237
|
+
flush() {
|
|
238
|
+
if (sending !== null)
|
|
239
|
+
return sending;
|
|
240
|
+
if (timer !== null) {
|
|
241
|
+
cancel(timer);
|
|
242
|
+
timer = null;
|
|
243
|
+
}
|
|
244
|
+
sending = send().finally(() => { sending = null; });
|
|
245
|
+
return sending;
|
|
246
|
+
},
|
|
247
|
+
flushOnHide() {
|
|
248
|
+
const batch = buffer.slice(0, BATCH);
|
|
249
|
+
if (batch.length === 0)
|
|
250
|
+
return;
|
|
251
|
+
options.transport(batch, { keepalive: true }).catch(() => undefined);
|
|
252
|
+
},
|
|
253
|
+
pending: () => [...buffer],
|
|
254
|
+
};
|
|
255
|
+
if (buffer.length > 0)
|
|
256
|
+
plan(0);
|
|
257
|
+
return client;
|
|
258
|
+
}
|
|
259
|
+
/** Receipts read from a collection answer, or an error when it is not one. */
|
|
260
|
+
export function readReceipts(body) {
|
|
261
|
+
if (!Array.isArray(body))
|
|
262
|
+
throw new TypeError('The collection answer is not a list of receipts');
|
|
263
|
+
return body.map(item => {
|
|
264
|
+
const row = typeof item === 'object' && item !== null ? item : {};
|
|
265
|
+
const status = row['status'];
|
|
266
|
+
if (status !== 'accepted' && status !== 'duplicate' && status !== 'conflict' && status !== 'invalid') {
|
|
267
|
+
throw new TypeError('A receipt has no known status');
|
|
268
|
+
}
|
|
269
|
+
return { event_id: typeof row['event_id'] === 'string' ? row['event_id'] : null, status,
|
|
270
|
+
received_at: typeof row['received_at'] === 'string' ? row['received_at'] : null };
|
|
271
|
+
});
|
|
272
|
+
}
|
|
273
|
+
/**
|
|
274
|
+
* The transport for a Supabase project (or any PostgREST): it calls skene.collect_events through the
|
|
275
|
+
* project's Data API with its key. In a browser use the public key: those events are recorded as sent
|
|
276
|
+
* from a browser. A 400 answer (the database could not read what the batch holds) throws a
|
|
277
|
+
* RefusedBatchError, so the batch is split; any other non-2xx answer or an unreadable body throws,
|
|
278
|
+
* so the batch is retried.
|
|
279
|
+
*/
|
|
280
|
+
export function supabaseTransport(config) {
|
|
281
|
+
const endpoint = new URL('rest/v1/rpc/collect_events', config.url.endsWith('/') ? config.url : `${config.url}/`).toString();
|
|
282
|
+
const call = config.fetch ?? globalThis.fetch.bind(globalThis);
|
|
283
|
+
return async (events, { keepalive }) => {
|
|
284
|
+
const response = await call(endpoint, {
|
|
285
|
+
method: 'POST', keepalive,
|
|
286
|
+
headers: { apikey: config.key, authorization: `Bearer ${config.key}`, 'content-type': 'application/json',
|
|
287
|
+
'content-profile': config.schema ?? 'skene', accept: 'application/json' },
|
|
288
|
+
body: JSON.stringify({ p_events: events }),
|
|
289
|
+
});
|
|
290
|
+
if (keepalive)
|
|
291
|
+
return [];
|
|
292
|
+
if (response.status === 400)
|
|
293
|
+
throw new RefusedBatchError('The collection call answered 400');
|
|
294
|
+
if (!response.ok)
|
|
295
|
+
throw new Error(`The collection call answered ${response.status}`);
|
|
296
|
+
return readReceipts(await response.json());
|
|
297
|
+
};
|
|
298
|
+
}
|
|
299
|
+
/** Browser storage: localStorage for the device's identities, sessionStorage for the tab's unsent buffer. */
|
|
300
|
+
export function browserStores() {
|
|
301
|
+
const wrap = (name) => ({
|
|
302
|
+
get: key => globalThis[name]?.getItem(key) ?? null,
|
|
303
|
+
set: (key, value) => { globalThis[name]?.setItem(key, value); },
|
|
304
|
+
});
|
|
305
|
+
return { deviceStore: wrap('localStorage'), bufferStore: wrap('sessionStorage') };
|
|
306
|
+
}
|
|
307
|
+
/** Sends the buffer with keepalive whenever the page is hidden; returns the function that stops it. */
|
|
308
|
+
export function sendOnPageHide(client, page = globalThis.document) {
|
|
309
|
+
const listener = () => { if (page.visibilityState === 'hidden')
|
|
310
|
+
client.flushOnHide(); };
|
|
311
|
+
page.addEventListener('visibilitychange', listener);
|
|
312
|
+
return () => page.removeEventListener('visibilitychange', listener);
|
|
313
|
+
}
|
package/kit/apply.sql
ADDED
|
@@ -0,0 +1,430 @@
|
|
|
1
|
+
-- Plan row L2-4 (LCH-03.sdk): the Skene Flows SDK route's change to the customer's own Postgres. The
|
|
2
|
+
-- customer applies it, as the role that will own schema `skene`, in the database the SDK writes to;
|
|
3
|
+
-- Skene never runs it and holds no credential that could. Rollback: rollback.sql beside it.
|
|
4
|
+
-- The rule record is docs/adr/20261010-flows-sdk-route-contract.md (revised by L2-4).
|
|
5
|
+
--
|
|
6
|
+
-- What it creates, all in schema `skene`:
|
|
7
|
+
-- events the collection table, append-only and keyed by event_id. Each row is one
|
|
8
|
+
-- event the SDK sent: its id, name and time, the person id after identify
|
|
9
|
+
-- or the anonymous id before it, the SDK installation and its sequence
|
|
10
|
+
-- number, up to 20 short properties, whether a browser sent it, and the
|
|
11
|
+
-- database's receive time.
|
|
12
|
+
-- collect_events(jsonb) the one way in: a batch of 1 to 100 events, answered per event after the
|
|
13
|
+
-- transaction commits: accepted (new), duplicate (an identical retry, with
|
|
14
|
+
-- the first receive time), conflict (the same id, or the same installation
|
|
15
|
+
-- sequence from the same origin, with other content; refused) or invalid
|
|
16
|
+
-- (refused). An event timed more than 5 minutes ahead of the database's
|
|
17
|
+
-- clock, or more than 7 days behind it (the SDK's maximum buffer age), is
|
|
18
|
+
-- invalid.
|
|
19
|
+
-- sequence_gaps each run of sequence numbers an installation skipped, per origin: an event
|
|
20
|
+
-- the SDK numbered that never arrived, with the time span it may cover.
|
|
21
|
+
-- measurement_events the read contract's relation over events.
|
|
22
|
+
-- measurement_packages, a sealed package: its manifest and its rows, which Skene's worker reads
|
|
23
|
+
-- measurement_package_events in one read-only snapshot and checks against the owner qualification's
|
|
24
|
+
-- row count and SHA-256.
|
|
25
|
+
-- seal_measurement_package(...) seals one closed interval into a package and returns the manifest to
|
|
26
|
+
-- give Skene, with how many of its rows a browser sent. It refuses an
|
|
27
|
+
-- interval a server installation's sequence gap overlaps, a row the worker
|
|
28
|
+
-- would refuse, or more than 50,000 rows. A browser installation's gap does
|
|
29
|
+
-- not refuse the seal, because anyone holding the public key can forge one;
|
|
30
|
+
-- the manifest reports how many overlap (browser_gaps) and which
|
|
31
|
+
-- installations (browser_gap_installations), for the owner to judge before
|
|
32
|
+
-- qualifying the package.
|
|
33
|
+
--
|
|
34
|
+
-- Who may call collect_events: on Supabase the kit grants it to anon and authenticated, the roles of
|
|
35
|
+
-- your project's public key, and to service_role, the role of its secret key; add `skene` to the
|
|
36
|
+
-- Data API's exposed schemas. An event whose caller is anon or authenticated is recorded as sent from
|
|
37
|
+
-- a browser: anyone holding the public key can send one, so send conversion-critical events from your
|
|
38
|
+
-- server. Nothing limits how many events the public key may send, so the table can grow without
|
|
39
|
+
-- bound; delete rows you no longer need as your retention. Anywhere else, grant it to your backend's
|
|
40
|
+
-- role yourself:
|
|
41
|
+
-- grant usage on schema skene to <backend>;
|
|
42
|
+
-- grant execute on function skene.collect_events(jsonb) to <backend>;
|
|
43
|
+
--
|
|
44
|
+
-- Not visible here: an event lost after an installation's last delivered event (a closed tab with
|
|
45
|
+
-- unsent events, a buffer that never reconnected) leaves no gap. Seal an interval only some time after
|
|
46
|
+
-- it ends, so retries have had time to arrive.
|
|
47
|
+
--
|
|
48
|
+
-- After applying, give your read-only Skene role exactly this, and nothing on skene.events:
|
|
49
|
+
-- grant usage on schema skene to <reader>;
|
|
50
|
+
-- grant select on skene.measurement_packages, skene.measurement_package_events to <reader>;
|
|
51
|
+
-- and register the destination with schema skene.
|
|
52
|
+
begin;
|
|
53
|
+
do $$
|
|
54
|
+
begin
|
|
55
|
+
if to_regnamespace('skene') is not null then
|
|
56
|
+
raise exception 'Schema skene already exists; the Flows SDK kit creates it';
|
|
57
|
+
end if;
|
|
58
|
+
end;
|
|
59
|
+
$$;
|
|
60
|
+
create schema skene;
|
|
61
|
+
revoke all on schema skene from public;
|
|
62
|
+
|
|
63
|
+
-- The worker's text rule: 1 to 200 characters, no C0 control character, and nothing JavaScript's
|
|
64
|
+
-- trim() would remove at either end.
|
|
65
|
+
create function skene.measurement_text_ok(v text, p_max integer default 200) returns boolean
|
|
66
|
+
language sql immutable set search_path = '' as $$
|
|
67
|
+
select coalesce(length(v) between 1 and p_max
|
|
68
|
+
and v = pg_catalog.btrim(v, E' \t\n\x0b\x0c\r
')
|
|
69
|
+
and v = pg_catalog.translate(v, E'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f'
|
|
70
|
+
|| E'\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f', ''), false)
|
|
71
|
+
$$;
|
|
72
|
+
|
|
73
|
+
-- One JSON value of a sealed entry property, written exactly as JSON.stringify writes it, or null
|
|
74
|
+
-- when the worker would refuse it: text of at most 200 characters, a safe integer or a boolean.
|
|
75
|
+
create function skene.measurement_property_json(v jsonb) returns text
|
|
76
|
+
language sql immutable set search_path = '' as $$
|
|
77
|
+
select case pg_catalog.jsonb_typeof(v)
|
|
78
|
+
when 'string' then case when length(v #>> '{}') <= 200 then pg_catalog.to_json(v #>> '{}')::text end
|
|
79
|
+
when 'boolean' then v::text
|
|
80
|
+
when 'number' then case when (v::text)::numeric = trunc((v::text)::numeric)
|
|
81
|
+
and abs((v::text)::numeric) <= 9007199254740991 then ((v::text)::numeric)::bigint::text end
|
|
82
|
+
end
|
|
83
|
+
$$;
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
-- A uuid written exactly as the SDK writes one (lowercase, hyphenated), or null.
|
|
87
|
+
create function skene.canonical_uuid(v jsonb) returns uuid
|
|
88
|
+
language plpgsql immutable set search_path = '' as $$
|
|
89
|
+
declare
|
|
90
|
+
parsed uuid;
|
|
91
|
+
begin
|
|
92
|
+
if v is null or pg_catalog.jsonb_typeof(v) <> 'string' then return null; end if;
|
|
93
|
+
begin
|
|
94
|
+
parsed := (v #>> '{}')::uuid;
|
|
95
|
+
exception when invalid_text_representation then
|
|
96
|
+
return null;
|
|
97
|
+
end;
|
|
98
|
+
return case when parsed::text = v #>> '{}' then parsed end;
|
|
99
|
+
end;
|
|
100
|
+
$$;
|
|
101
|
+
|
|
102
|
+
-- An event time written exactly as JavaScript's Date.prototype.toISOString writes one, or null.
|
|
103
|
+
create function skene.canonical_time(v jsonb) returns timestamptz
|
|
104
|
+
language plpgsql immutable set search_path = '' as $$
|
|
105
|
+
declare
|
|
106
|
+
parsed timestamptz;
|
|
107
|
+
begin
|
|
108
|
+
if v is null or pg_catalog.jsonb_typeof(v) <> 'string' then return null; end if;
|
|
109
|
+
begin
|
|
110
|
+
parsed := (v #>> '{}')::timestamptz;
|
|
111
|
+
exception when invalid_datetime_format or datetime_field_overflow then
|
|
112
|
+
return null;
|
|
113
|
+
end;
|
|
114
|
+
if not pg_catalog.isfinite(parsed) or parsed < 'epoch'::timestamptz then return null; end if;
|
|
115
|
+
return case when pg_catalog.to_char(parsed at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') = v #>> '{}'
|
|
116
|
+
then parsed end;
|
|
117
|
+
end;
|
|
118
|
+
$$;
|
|
119
|
+
|
|
120
|
+
-- An event's properties: none, or an object of 1 to 20 keys of at most 120 characters whose values
|
|
121
|
+
-- the worker can seal (short text, a safe integer or a boolean).
|
|
122
|
+
create function skene.properties_ok(v jsonb) returns boolean
|
|
123
|
+
language sql immutable set search_path = '' as $$
|
|
124
|
+
select v is null or pg_catalog.jsonb_typeof(v) = 'object'
|
|
125
|
+
and (select count(*) from pg_catalog.jsonb_each(v)) between 1 and 20
|
|
126
|
+
and not exists (select 1 from pg_catalog.jsonb_each(v) p
|
|
127
|
+
where not skene.measurement_text_ok(p.key, 120) or skene.measurement_property_json(p.value) is null)
|
|
128
|
+
$$;
|
|
129
|
+
|
|
130
|
+
create table skene.events (
|
|
131
|
+
event_id uuid primary key,
|
|
132
|
+
event_name text not null check (skene.measurement_text_ok(event_name)),
|
|
133
|
+
occurred_at timestamptz not null
|
|
134
|
+
check (occurred_at >= 'epoch'::timestamptz and date_trunc('milliseconds', occurred_at) = occurred_at),
|
|
135
|
+
person_id text check (person_id is null or skene.measurement_text_ok(person_id)),
|
|
136
|
+
anonymous_id text check (anonymous_id is null or skene.measurement_text_ok(anonymous_id)),
|
|
137
|
+
installation_id uuid not null,
|
|
138
|
+
sequence bigint not null check (sequence between 1 and 9007199254740991),
|
|
139
|
+
properties jsonb check (skene.properties_ok(properties)),
|
|
140
|
+
origin text not null check (origin in ('browser', 'server')),
|
|
141
|
+
received_at timestamptz not null default pg_catalog.now(),
|
|
142
|
+
check (num_nonnulls(person_id, anonymous_id) = 1),
|
|
143
|
+
-- Per origin, so an event sent with the public key can never take a server installation's number.
|
|
144
|
+
unique (installation_id, origin, sequence)
|
|
145
|
+
);
|
|
146
|
+
|
|
147
|
+
-- Append-only: a row is never changed. Deleting rows is the owner's retention, and expires any
|
|
148
|
+
-- package sealed over them only if the package itself is deleted.
|
|
149
|
+
create function skene.refuse_event_update() returns trigger
|
|
150
|
+
language plpgsql set search_path = '' as $$
|
|
151
|
+
begin
|
|
152
|
+
raise exception 'skene.events is append-only';
|
|
153
|
+
end;
|
|
154
|
+
$$;
|
|
155
|
+
create trigger events_append_only before update on skene.events
|
|
156
|
+
for each row execute function skene.refuse_event_update();
|
|
157
|
+
|
|
158
|
+
-- The one way in. See the header for its answers. A malformed event is answered invalid and the rest
|
|
159
|
+
-- of the batch still lands; a malformed batch is refused whole.
|
|
160
|
+
create function skene.collect_events(p_events jsonb)
|
|
161
|
+
returns table (event_id uuid, status text, received_at timestamptz)
|
|
162
|
+
language plpgsql security definer set search_path = '' as $$
|
|
163
|
+
#variable_conflict use_column
|
|
164
|
+
declare
|
|
165
|
+
v_origin text := case when pg_catalog.current_setting('role', true) in ('anon', 'authenticated')
|
|
166
|
+
then 'browser' else 'server' end;
|
|
167
|
+
v_item jsonb;
|
|
168
|
+
v_id uuid;
|
|
169
|
+
v_installation uuid;
|
|
170
|
+
v_at timestamptz;
|
|
171
|
+
v_name text;
|
|
172
|
+
v_person text;
|
|
173
|
+
v_anonymous text;
|
|
174
|
+
v_sequence bigint;
|
|
175
|
+
v_properties jsonb;
|
|
176
|
+
v_received timestamptz;
|
|
177
|
+
v_first skene.events%rowtype;
|
|
178
|
+
begin
|
|
179
|
+
if p_events is null or pg_catalog.jsonb_typeof(p_events) <> 'array'
|
|
180
|
+
or pg_catalog.jsonb_array_length(p_events) not between 1 and 100 then
|
|
181
|
+
raise exception 'A batch is a JSON array of 1 to 100 events' using errcode = '22023';
|
|
182
|
+
end if;
|
|
183
|
+
for v_item in select e.value from pg_catalog.jsonb_array_elements(p_events) e loop
|
|
184
|
+
v_id := case when pg_catalog.jsonb_typeof(v_item) = 'object' then skene.canonical_uuid(v_item -> 'event_id') end;
|
|
185
|
+
if v_id is null
|
|
186
|
+
or exists (select 1 from pg_catalog.jsonb_object_keys(v_item) k where k not in ('event_id', 'event_name',
|
|
187
|
+
'occurred_at', 'person_id', 'anonymous_id', 'installation_id', 'sequence', 'properties'))
|
|
188
|
+
or pg_catalog.jsonb_typeof(v_item -> 'event_name') is distinct from 'string'
|
|
189
|
+
or pg_catalog.jsonb_typeof(v_item -> 'sequence') is distinct from 'number'
|
|
190
|
+
or coalesce(pg_catalog.jsonb_typeof(v_item -> 'person_id'), 'null') not in ('string', 'null')
|
|
191
|
+
or coalesce(pg_catalog.jsonb_typeof(v_item -> 'anonymous_id'), 'null') not in ('string', 'null')
|
|
192
|
+
or coalesce(pg_catalog.jsonb_typeof(v_item -> 'properties'), 'null') not in ('object', 'null') then
|
|
193
|
+
event_id := v_id; status := 'invalid'; received_at := null;
|
|
194
|
+
return next;
|
|
195
|
+
continue;
|
|
196
|
+
end if;
|
|
197
|
+
v_name := v_item ->> 'event_name';
|
|
198
|
+
v_at := skene.canonical_time(v_item -> 'occurred_at');
|
|
199
|
+
v_installation := skene.canonical_uuid(v_item -> 'installation_id');
|
|
200
|
+
v_person := v_item ->> 'person_id';
|
|
201
|
+
v_anonymous := v_item ->> 'anonymous_id';
|
|
202
|
+
v_properties := case when pg_catalog.jsonb_typeof(v_item -> 'properties') = 'object' then v_item -> 'properties' end;
|
|
203
|
+
v_sequence := case when (v_item ->> 'sequence')::numeric = trunc((v_item ->> 'sequence')::numeric)
|
|
204
|
+
and (v_item ->> 'sequence')::numeric between 1 and 9007199254740991 then (v_item ->> 'sequence')::bigint end;
|
|
205
|
+
if v_at is null or v_installation is null or v_sequence is null or not skene.measurement_text_ok(v_name)
|
|
206
|
+
or num_nonnulls(v_person, v_anonymous) <> 1
|
|
207
|
+
or v_person is not null and not skene.measurement_text_ok(v_person)
|
|
208
|
+
or v_anonymous is not null and not skene.measurement_text_ok(v_anonymous)
|
|
209
|
+
or not skene.properties_ok(v_properties) then
|
|
210
|
+
event_id := v_id; status := 'invalid'; received_at := null;
|
|
211
|
+
return next;
|
|
212
|
+
continue;
|
|
213
|
+
end if;
|
|
214
|
+
-- The table's two keys do the deduplication: a new id lands, anything else is compared. Only an
|
|
215
|
+
-- event timed within the window lands: at most 5 minutes ahead of the database's clock and at most
|
|
216
|
+
-- 7 days behind it, the SDK's maximum buffer age. Outside it, an identical retry of an event
|
|
217
|
+
-- stored earlier is still a duplicate, and anything else is invalid.
|
|
218
|
+
v_received := null;
|
|
219
|
+
if v_at between pg_catalog.now() - interval '7 days' and pg_catalog.now() + interval '5 minutes' then
|
|
220
|
+
insert into skene.events as e (event_id, event_name, occurred_at, person_id, anonymous_id, installation_id,
|
|
221
|
+
sequence, properties, origin)
|
|
222
|
+
values (v_id, v_name, v_at, v_person, v_anonymous, v_installation, v_sequence, v_properties, v_origin)
|
|
223
|
+
on conflict do nothing
|
|
224
|
+
returning e.received_at into v_received;
|
|
225
|
+
end if;
|
|
226
|
+
if v_received is not null then
|
|
227
|
+
event_id := v_id; status := 'accepted'; received_at := v_received;
|
|
228
|
+
return next;
|
|
229
|
+
continue;
|
|
230
|
+
end if;
|
|
231
|
+
select * into v_first from skene.events e where e.event_id = v_id;
|
|
232
|
+
if found and (v_first.event_name, v_first.occurred_at, v_first.person_id, v_first.anonymous_id,
|
|
233
|
+
v_first.installation_id, v_first.sequence, v_first.properties)
|
|
234
|
+
is not distinct from (v_name, v_at, v_person, v_anonymous, v_installation, v_sequence, v_properties) then
|
|
235
|
+
event_id := v_id; status := 'duplicate'; received_at := v_first.received_at;
|
|
236
|
+
elsif v_at not between pg_catalog.now() - interval '7 days' and pg_catalog.now() + interval '5 minutes' then
|
|
237
|
+
event_id := v_id; status := 'invalid'; received_at := null;
|
|
238
|
+
else
|
|
239
|
+
event_id := v_id; status := 'conflict'; received_at := null;
|
|
240
|
+
end if;
|
|
241
|
+
return next;
|
|
242
|
+
end loop;
|
|
243
|
+
end;
|
|
244
|
+
$$;
|
|
245
|
+
|
|
246
|
+
-- Each run of sequence numbers an installation skipped, per origin. The missing events were numbered
|
|
247
|
+
-- between the two delivered events around the run, so they happened between those two times (either
|
|
248
|
+
-- order, if the device's clock moved back). A run before an installation's first delivered event has
|
|
249
|
+
-- no earlier event. The span a gap may cover (gap_from to gap_to) is also held to what its events'
|
|
250
|
+
-- receive times allow: the kit refuses an event timed more than 7 days before it arrived or more than
|
|
251
|
+
-- 5 minutes after, so a missing event that could still have landed lies within those bounds.
|
|
252
|
+
create view skene.sequence_gaps as
|
|
253
|
+
select s.installation_id, s.origin, s.previous_sequence, s.sequence as next_sequence,
|
|
254
|
+
s.sequence - coalesce(s.previous_sequence, 0) - 1 as missing,
|
|
255
|
+
s.previous_occurred_at as after_occurred_at, s.occurred_at as before_occurred_at,
|
|
256
|
+
case when s.previous_sequence is null then s.received_at - interval '7 days'
|
|
257
|
+
else greatest(least(s.previous_occurred_at, s.occurred_at), least(s.previous_received_at, s.received_at) - interval '7 days')
|
|
258
|
+
end as gap_from,
|
|
259
|
+
least(greatest(s.previous_occurred_at, s.occurred_at),
|
|
260
|
+
greatest(s.previous_received_at, s.received_at) + interval '5 minutes') as gap_to
|
|
261
|
+
from (select e.installation_id, e.origin, e.sequence, e.occurred_at, e.received_at,
|
|
262
|
+
lag(e.sequence) over w as previous_sequence, lag(e.occurred_at) over w as previous_occurred_at,
|
|
263
|
+
lag(e.received_at) over w as previous_received_at
|
|
264
|
+
from skene.events e window w as (partition by e.installation_id, e.origin order by e.sequence)) s
|
|
265
|
+
where coalesce(s.previous_sequence, 0) + 1 < s.sequence;
|
|
266
|
+
|
|
267
|
+
-- The read contract (docs/adr/20261005-destination-read-contract.md), plus properties and origin.
|
|
268
|
+
create view skene.measurement_events as
|
|
269
|
+
select e.event_id::text as event_id, e.person_id, e.anonymous_id, e.event_name, e.occurred_at,
|
|
270
|
+
e.properties, e.origin
|
|
271
|
+
from skene.events e;
|
|
272
|
+
|
|
273
|
+
create table skene.measurement_packages (
|
|
274
|
+
snapshot_reference text primary key,
|
|
275
|
+
source_id text not null,
|
|
276
|
+
query_contract text not null check (query_contract in ('postgres-package-0001', 'postgres-package-0002')),
|
|
277
|
+
from_inclusive_ms bigint not null check (from_inclusive_ms >= 0),
|
|
278
|
+
to_exclusive_ms bigint not null,
|
|
279
|
+
event_names text[] not null,
|
|
280
|
+
row_count integer not null check (row_count between 0 and 50000),
|
|
281
|
+
rows_sha256 text not null check (length(rows_sha256) = 64),
|
|
282
|
+
browser_rows integer not null check (browser_rows between 0 and row_count),
|
|
283
|
+
browser_gaps integer not null default 0 check (browser_gaps >= 0),
|
|
284
|
+
browser_gap_installations uuid[] not null default '{}',
|
|
285
|
+
sealed_at timestamptz not null default now(),
|
|
286
|
+
check (from_inclusive_ms < to_exclusive_ms)
|
|
287
|
+
);
|
|
288
|
+
create table skene.measurement_package_events (
|
|
289
|
+
snapshot_reference text not null references skene.measurement_packages(snapshot_reference)
|
|
290
|
+
on delete cascade deferrable initially deferred,
|
|
291
|
+
event_id text not null,
|
|
292
|
+
person_id text,
|
|
293
|
+
anonymous_id text,
|
|
294
|
+
event_name text not null,
|
|
295
|
+
occurred_at timestamptz not null,
|
|
296
|
+
entry_properties jsonb,
|
|
297
|
+
origin text not null,
|
|
298
|
+
primary key (snapshot_reference, event_id)
|
|
299
|
+
);
|
|
300
|
+
|
|
301
|
+
-- Seal one closed interval [p_from, p_to) of the declared events into a package, and return the
|
|
302
|
+
-- manifest Skene's owner qualification must match, with how many rows a browser sent and which event
|
|
303
|
+
-- names they carry, and how many browser installations' sequence gaps overlap the interval and whose. Without p_entry_property_keys the package is postgres-package-0001; with them it is
|
|
304
|
+
-- 0002 and each row seals those keys of its properties. The digest is the worker's
|
|
305
|
+
-- (lib/measurement-sources/qualified-package.ts): one line per row, ordered by occurred_at, event_id in
|
|
306
|
+
-- C collation, each JSON.stringify([event_id, person_id, anonymous_id, event_name, occurred_at_ms(,
|
|
307
|
+
-- pairs)]) followed by a line feed.
|
|
308
|
+
create function skene.seal_measurement_package(p_snapshot_reference text, p_source_id text,
|
|
309
|
+
p_from timestamptz, p_to timestamptz, p_event_names text[], p_entry_property_keys text[] default null)
|
|
310
|
+
returns table (source_id text, query_contract text, snapshot_reference text, from_inclusive_ms bigint,
|
|
311
|
+
to_exclusive_ms bigint, event_names text[], row_count integer, rows_sha256 text, browser_rows integer,
|
|
312
|
+
browser_event_names text[], browser_gaps integer, browser_gap_installations uuid[], sealed_at timestamptz)
|
|
313
|
+
language plpgsql security invoker set search_path = '' as $$
|
|
314
|
+
declare
|
|
315
|
+
v_from_ms bigint;
|
|
316
|
+
v_to_ms bigint;
|
|
317
|
+
v_contract text := case when p_entry_property_keys is null then 'postgres-package-0001' else 'postgres-package-0002' end;
|
|
318
|
+
v_count integer;
|
|
319
|
+
v_problem bigint;
|
|
320
|
+
v_browser_gaps bigint;
|
|
321
|
+
v_browser_installations uuid[];
|
|
322
|
+
v_digest text;
|
|
323
|
+
begin
|
|
324
|
+
if not skene.measurement_text_ok(p_snapshot_reference) or not skene.measurement_text_ok(p_source_id) then
|
|
325
|
+
raise exception 'A package needs a snapshot reference and a source id of 1 to 200 characters, trimmed, without control characters';
|
|
326
|
+
end if;
|
|
327
|
+
if p_from is null or p_to is null or not isfinite(p_from) or not isfinite(p_to) or p_from >= p_to
|
|
328
|
+
or date_trunc('milliseconds', p_from) <> p_from or date_trunc('milliseconds', p_to) <> p_to
|
|
329
|
+
or p_from < 'epoch'::timestamptz then
|
|
330
|
+
raise exception 'A package needs a closed interval from before to after, on whole milliseconds, not before 1970';
|
|
331
|
+
end if;
|
|
332
|
+
v_from_ms := (extract(epoch from p_from) * 1000)::bigint;
|
|
333
|
+
v_to_ms := (extract(epoch from p_to) * 1000)::bigint;
|
|
334
|
+
if p_event_names is null or cardinality(p_event_names) not between 2 and 20
|
|
335
|
+
or exists (select 1 from unnest(p_event_names) n where not skene.measurement_text_ok(n))
|
|
336
|
+
or p_event_names is distinct from (select array_agg(distinct n collate "C" order by n collate "C") from unnest(p_event_names) n) then
|
|
337
|
+
raise exception 'A package needs 2 to 20 distinct event names, sorted by code point';
|
|
338
|
+
end if;
|
|
339
|
+
if p_entry_property_keys is not null and (cardinality(p_entry_property_keys) not between 1 and 20
|
|
340
|
+
or exists (select 1 from unnest(p_entry_property_keys) k where not skene.measurement_text_ok(k, 120))
|
|
341
|
+
or (select count(distinct k) from unnest(p_entry_property_keys) k) <> cardinality(p_entry_property_keys)) then
|
|
342
|
+
raise exception 'Entry property keys are 1 to 20 distinct names of at most 120 characters';
|
|
343
|
+
end if;
|
|
344
|
+
if exists (select 1 from skene.measurement_packages m where m.snapshot_reference = p_snapshot_reference) then
|
|
345
|
+
raise exception 'Snapshot % is already sealed; a package is never resealed', p_snapshot_reference;
|
|
346
|
+
end if;
|
|
347
|
+
-- An incomplete interval is never sealed: a numbered event that never arrived may have happened in it.
|
|
348
|
+
-- Only a server installation's gap refuses: anyone holding the public key can forge a browser gap,
|
|
349
|
+
-- so those are reported in the manifest for the owner to judge instead.
|
|
350
|
+
select count(*) filter (where g.origin = 'server'), count(*) filter (where g.origin = 'browser'),
|
|
351
|
+
coalesce(array_agg(distinct g.installation_id order by g.installation_id) filter (where g.origin = 'browser'), '{}')
|
|
352
|
+
into v_problem, v_browser_gaps, v_browser_installations
|
|
353
|
+
from skene.sequence_gaps g where g.gap_from < p_to and g.gap_to >= p_from;
|
|
354
|
+
if v_problem > 0 then
|
|
355
|
+
raise exception '% server sequence gaps overlap this interval, so it is incomplete; it was not sealed', v_problem;
|
|
356
|
+
end if;
|
|
357
|
+
insert into skene.measurement_package_events(snapshot_reference, event_id, person_id, anonymous_id,
|
|
358
|
+
event_name, occurred_at, entry_properties, origin)
|
|
359
|
+
select p_snapshot_reference, m.event_id, m.person_id, m.anonymous_id, m.event_name, m.occurred_at,
|
|
360
|
+
case when p_entry_property_keys is null then null
|
|
361
|
+
else (select jsonb_object_agg(k, m.properties -> k) from unnest(p_entry_property_keys) k
|
|
362
|
+
where m.properties ? k and jsonb_typeof(m.properties -> k) <> 'null') end,
|
|
363
|
+
m.origin
|
|
364
|
+
from skene.measurement_events m
|
|
365
|
+
where m.event_name = any(p_event_names) and m.occurred_at >= p_from and m.occurred_at < p_to;
|
|
366
|
+
get diagnostics v_count = row_count;
|
|
367
|
+
if v_count > 50000 then
|
|
368
|
+
raise exception 'The interval holds % rows; a package holds at most 50,000', v_count;
|
|
369
|
+
end if;
|
|
370
|
+
select count(*) into v_problem from skene.measurement_package_events r
|
|
371
|
+
where r.snapshot_reference = p_snapshot_reference and (not skene.measurement_text_ok(r.event_id)
|
|
372
|
+
or r.person_id is not null and not skene.measurement_text_ok(r.person_id)
|
|
373
|
+
or r.anonymous_id is not null and not skene.measurement_text_ok(r.anonymous_id)
|
|
374
|
+
or exists (select 1 from jsonb_each(coalesce(r.entry_properties, '{}'::jsonb)) p
|
|
375
|
+
where skene.measurement_property_json(p.value) is null));
|
|
376
|
+
if v_problem > 0 then
|
|
377
|
+
raise exception '% rows carry an id over 200 characters, padded or with a control character, or an entry property that is not short text, a safe integer or a boolean; nothing was sealed', v_problem;
|
|
378
|
+
end if;
|
|
379
|
+
select encode(sha256(convert_to(coalesce(string_agg(
|
|
380
|
+
'[' || to_json(r.event_id)::text || ',' || coalesce(to_json(r.person_id)::text, 'null')
|
|
381
|
+
|| ',' || coalesce(to_json(r.anonymous_id)::text, 'null') || ',' || to_json(r.event_name)::text
|
|
382
|
+
|| ',' || trunc(extract(epoch from r.occurred_at) * 1000)::bigint::text
|
|
383
|
+
|| case when v_contract = 'postgres-package-0001' then ''
|
|
384
|
+
else ',' || coalesce((select '[' || string_agg('[' || to_json(p.key)::text || ','
|
|
385
|
+
|| skene.measurement_property_json(p.value) || ']', ',' order by p.key collate "C") || ']'
|
|
386
|
+
from jsonb_each(r.entry_properties) p), 'null') end
|
|
387
|
+
|| ']' || E'\n', '' order by r.occurred_at, r.event_id collate "C"), ''), 'UTF8')), 'hex')
|
|
388
|
+
into v_digest
|
|
389
|
+
from skene.measurement_package_events r where r.snapshot_reference = p_snapshot_reference;
|
|
390
|
+
insert into skene.measurement_packages(snapshot_reference, source_id, query_contract, from_inclusive_ms,
|
|
391
|
+
to_exclusive_ms, event_names, row_count, rows_sha256, browser_rows, browser_gaps, browser_gap_installations)
|
|
392
|
+
values (p_snapshot_reference, p_source_id, v_contract, v_from_ms, v_to_ms, p_event_names, v_count, v_digest,
|
|
393
|
+
(select count(*) from skene.measurement_package_events r
|
|
394
|
+
where r.snapshot_reference = p_snapshot_reference and r.origin = 'browser'),
|
|
395
|
+
v_browser_gaps, v_browser_installations);
|
|
396
|
+
return query select m.source_id, m.query_contract, m.snapshot_reference, m.from_inclusive_ms, m.to_exclusive_ms,
|
|
397
|
+
m.event_names, m.row_count, m.rows_sha256, m.browser_rows,
|
|
398
|
+
coalesce((select array_agg(distinct r.event_name collate "C" order by r.event_name collate "C")
|
|
399
|
+
from skene.measurement_package_events r
|
|
400
|
+
where r.snapshot_reference = p_snapshot_reference and r.origin = 'browser'), '{}'::text[]),
|
|
401
|
+
m.browser_gaps, m.browser_gap_installations, m.sealed_at from skene.measurement_packages m
|
|
402
|
+
where m.snapshot_reference = p_snapshot_reference;
|
|
403
|
+
end;
|
|
404
|
+
$$;
|
|
405
|
+
|
|
406
|
+
revoke all on function skene.collect_events(jsonb) from public;
|
|
407
|
+
revoke all on function skene.seal_measurement_package(text, text, timestamptz, timestamptz, text[], text[]) from public;
|
|
408
|
+
revoke all on function skene.measurement_text_ok(text, integer) from public;
|
|
409
|
+
revoke all on function skene.measurement_property_json(jsonb) from public;
|
|
410
|
+
revoke all on function skene.canonical_uuid(jsonb) from public;
|
|
411
|
+
revoke all on function skene.canonical_time(jsonb) from public;
|
|
412
|
+
revoke all on function skene.properties_ok(jsonb) from public;
|
|
413
|
+
revoke all on function skene.refuse_event_update() from public;
|
|
414
|
+
revoke all on skene.events, skene.sequence_gaps, skene.measurement_events, skene.measurement_packages,
|
|
415
|
+
skene.measurement_package_events from public;
|
|
416
|
+
|
|
417
|
+
-- On Supabase: the public key's roles and the secret key's role may call collect_events, nothing else.
|
|
418
|
+
do $$
|
|
419
|
+
declare
|
|
420
|
+
r text;
|
|
421
|
+
begin
|
|
422
|
+
foreach r in array array['anon', 'authenticated', 'service_role'] loop
|
|
423
|
+
if exists (select 1 from pg_catalog.pg_roles where rolname = r) then
|
|
424
|
+
execute format('grant usage on schema skene to %I', r);
|
|
425
|
+
execute format('grant execute on function skene.collect_events(jsonb) to %I', r);
|
|
426
|
+
end if;
|
|
427
|
+
end loop;
|
|
428
|
+
end;
|
|
429
|
+
$$;
|
|
430
|
+
commit;
|
package/kit/rollback.sql
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
-- Rollback of apply.sql, run by the same role in the same database. The collected events and the
|
|
2
|
+
-- sealed packages are the customer's records: the rollback refuses while either holds a row, so
|
|
3
|
+
-- nothing is discarded by it. Move or delete them first if they should go (deleting a package expires
|
|
4
|
+
-- its snapshot: a measurement bound to it then reports the snapshot as expired, never a read of
|
|
5
|
+
-- current rows). Then it removes schema skene and everything the kit created in it.
|
|
6
|
+
begin;
|
|
7
|
+
do $$
|
|
8
|
+
begin
|
|
9
|
+
if exists (select 1 from skene.measurement_packages) or exists (select 1 from skene.events) then
|
|
10
|
+
raise exception 'The Flows SDK kit rollback keeps collected events and sealed packages; move or delete them first';
|
|
11
|
+
end if;
|
|
12
|
+
end;
|
|
13
|
+
$$;
|
|
14
|
+
drop function skene.seal_measurement_package(text, text, timestamptz, timestamptz, text[], text[]);
|
|
15
|
+
drop table skene.measurement_package_events;
|
|
16
|
+
drop table skene.measurement_packages;
|
|
17
|
+
drop view skene.measurement_events;
|
|
18
|
+
drop view skene.sequence_gaps;
|
|
19
|
+
drop function skene.collect_events(jsonb);
|
|
20
|
+
drop table skene.events;
|
|
21
|
+
drop function skene.refuse_event_update();
|
|
22
|
+
drop function skene.properties_ok(jsonb);
|
|
23
|
+
drop function skene.canonical_time(jsonb);
|
|
24
|
+
drop function skene.canonical_uuid(jsonb);
|
|
25
|
+
drop function skene.measurement_property_json(jsonb);
|
|
26
|
+
drop function skene.measurement_text_ok(text, integer);
|
|
27
|
+
drop schema skene;
|
|
28
|
+
commit;
|
package/package.json
CHANGED
|
@@ -1,6 +1,32 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@skene/flows-sdk",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.1.1",
|
|
4
|
+
"description": "Skene Flows SDK: numbers, buffers and writes product events into your own database through the Flows SDK kit's skene.collect_events.",
|
|
5
|
+
"license": "SEE LICENSE IN LICENSE",
|
|
6
|
+
"type": "module",
|
|
7
|
+
"exports": {
|
|
8
|
+
".": {
|
|
9
|
+
"types": "./dist/index.d.ts",
|
|
10
|
+
"default": "./dist/index.js"
|
|
11
|
+
}
|
|
12
|
+
},
|
|
13
|
+
"files": [
|
|
14
|
+
"dist",
|
|
15
|
+
"kit",
|
|
16
|
+
"LICENSE"
|
|
17
|
+
],
|
|
18
|
+
"sideEffects": false,
|
|
19
|
+
"engines": {
|
|
20
|
+
"node": ">=20"
|
|
21
|
+
},
|
|
22
|
+
"repository": {
|
|
23
|
+
"type": "git",
|
|
24
|
+
"url": "git+https://github.com/SkeneTechnologies/skene-cloud-rebuild.git",
|
|
25
|
+
"directory": "packages/flows-sdk"
|
|
26
|
+
},
|
|
27
|
+
"publishConfig": {
|
|
28
|
+
"access": "public",
|
|
29
|
+
"registry": "https://registry.npmjs.org",
|
|
30
|
+
"provenance": false
|
|
31
|
+
}
|
|
32
|
+
}
|