@spooky-sync/core 0.0.1-canary.8 → 0.0.1-canary.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +56 -0
- package/dist/index.d.ts +854 -339
- package/dist/index.js +2633 -396
- package/dist/otel/index.d.ts +21 -0
- package/dist/otel/index.js +86 -0
- package/dist/types.d.ts +494 -0
- package/package.json +39 -9
- package/skills/sp00ky-core/SKILL.md +258 -0
- package/skills/sp00ky-core/references/auth.md +98 -0
- package/skills/sp00ky-core/references/config.md +76 -0
- package/src/build-globals.d.ts +12 -0
- package/src/events/events.test.ts +2 -1
- package/src/events/index.ts +3 -0
- package/src/index.ts +9 -2
- package/src/modules/auth/events/index.ts +2 -1
- package/src/modules/auth/index.ts +59 -20
- package/src/modules/cache/index.ts +41 -29
- package/src/modules/cache/types.ts +2 -2
- package/src/modules/crdt/crdt-field.ts +281 -0
- package/src/modules/crdt/crdt-hydration.test.ts +206 -0
- package/src/modules/crdt/index.ts +352 -0
- package/src/modules/data/data.status.test.ts +108 -0
- package/src/modules/data/index.ts +726 -108
- package/src/modules/data/window-query.test.ts +52 -0
- package/src/modules/data/window-query.ts +130 -0
- package/src/modules/devtools/index.ts +89 -21
- package/src/modules/devtools/versions.test.ts +74 -0
- package/src/modules/devtools/versions.ts +81 -0
- package/src/modules/feature-flag/index.ts +166 -0
- package/src/modules/ref-tables.test.ts +56 -0
- package/src/modules/ref-tables.ts +57 -0
- package/src/modules/sync/engine.ts +97 -37
- package/src/modules/sync/events/index.ts +3 -2
- package/src/modules/sync/queue/queue-down.ts +5 -4
- package/src/modules/sync/queue/queue-up.ts +14 -13
- package/src/modules/sync/scheduler.ts +2 -2
- package/src/modules/sync/sync.ts +676 -58
- package/src/modules/sync/utils.test.ts +269 -2
- package/src/modules/sync/utils.ts +182 -17
- package/src/otel/index.ts +127 -0
- package/src/services/database/database.ts +11 -11
- package/src/services/database/events/index.ts +2 -1
- package/src/services/database/local-migrator.ts +21 -21
- package/src/services/database/local.test.ts +33 -0
- package/src/services/database/local.ts +192 -32
- package/src/services/database/remote.ts +13 -13
- package/src/services/logger/index.ts +6 -101
- package/src/services/persistence/localstorage.ts +2 -2
- package/src/services/persistence/resilient.ts +41 -0
- package/src/services/persistence/surrealdb.ts +9 -9
- package/src/services/stream-processor/index.ts +248 -37
- package/src/services/stream-processor/permissions.test.ts +47 -0
- package/src/services/stream-processor/permissions.ts +53 -0
- package/src/services/stream-processor/stream-processor.batch.test.ts +136 -0
- package/src/services/stream-processor/stream-processor.test.ts +1 -1
- package/src/services/stream-processor/wasm-types.ts +18 -2
- package/src/sp00ky.auth-order.test.ts +65 -0
- package/src/sp00ky.ts +625 -0
- package/src/types.ts +140 -13
- package/src/utils/index.ts +35 -13
- package/src/utils/parser.ts +3 -2
- package/src/utils/surql.ts +24 -15
- package/src/utils/withRetry.test.ts +1 -1
- package/tsdown.config.ts +55 -1
- package/src/spooky.ts +0 -392
|
@@ -1,114 +1,19 @@
|
|
|
1
|
-
import pino, { Level, type Logger as PinoLogger, type LoggerOptions } from 'pino';
|
|
2
|
-
import {
|
|
3
|
-
import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-proto';
|
|
4
|
-
import { resourceFromAttributes } from '@opentelemetry/resources';
|
|
5
|
-
import { ATTR_SERVICE_NAME } from '@opentelemetry/semantic-conventions';
|
|
6
|
-
import { createContextKey } from '@opentelemetry/api';
|
|
7
|
-
|
|
8
|
-
const CATEGORY_KEY = createContextKey('Category');
|
|
1
|
+
import pino, { type Level, type Logger as PinoLogger, type LoggerOptions } from 'pino';
|
|
2
|
+
import type { PinoTransmit } from '../../types';
|
|
9
3
|
|
|
10
4
|
export type Logger = PinoLogger;
|
|
11
5
|
|
|
12
|
-
|
|
13
|
-
// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#severity-fields
|
|
14
|
-
function mapLevelToSeverityNumber(level: string): number {
|
|
15
|
-
switch (level) {
|
|
16
|
-
case 'trace':
|
|
17
|
-
return 1;
|
|
18
|
-
case 'debug':
|
|
19
|
-
return 5;
|
|
20
|
-
case 'info':
|
|
21
|
-
return 9;
|
|
22
|
-
case 'warn':
|
|
23
|
-
return 13;
|
|
24
|
-
case 'error':
|
|
25
|
-
return 17;
|
|
26
|
-
case 'fatal':
|
|
27
|
-
return 21;
|
|
28
|
-
default:
|
|
29
|
-
return 9;
|
|
30
|
-
}
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
export function createLogger(level: Level = 'info', otelEndpoint?: string): Logger {
|
|
6
|
+
export function createLogger(level: Level = 'info', transmit?: PinoTransmit): Logger {
|
|
34
7
|
const browserConfig: LoggerOptions['browser'] = {
|
|
35
8
|
asObject: true,
|
|
36
9
|
write: (o: any) => {
|
|
10
|
+
// oxlint-disable-next-line no-console
|
|
37
11
|
console.log(JSON.stringify(o));
|
|
38
12
|
},
|
|
39
13
|
};
|
|
40
14
|
|
|
41
|
-
if (
|
|
42
|
-
|
|
43
|
-
const resource = resourceFromAttributes({
|
|
44
|
-
[ATTR_SERVICE_NAME]: 'spooky-client',
|
|
45
|
-
});
|
|
46
|
-
|
|
47
|
-
const exporter = new OTLPLogExporter({
|
|
48
|
-
url: otelEndpoint,
|
|
49
|
-
});
|
|
50
|
-
|
|
51
|
-
// Pass processors in constructor as this SDK version requires it
|
|
52
|
-
const loggerProvider = new LoggerProvider({
|
|
53
|
-
resource,
|
|
54
|
-
processors: [new BatchLogRecordProcessor(exporter)],
|
|
55
|
-
});
|
|
56
|
-
|
|
57
|
-
const otelLogger: Record<string, ReturnType<typeof loggerProvider.getLogger>> = {};
|
|
58
|
-
|
|
59
|
-
const getOtelLogger = (category: string) => {
|
|
60
|
-
if (!otelLogger[category]) {
|
|
61
|
-
otelLogger[category] = loggerProvider.getLogger(category);
|
|
62
|
-
}
|
|
63
|
-
return otelLogger[category];
|
|
64
|
-
};
|
|
65
|
-
|
|
66
|
-
browserConfig.transmit = {
|
|
67
|
-
level: level,
|
|
68
|
-
send: (levelLabel: string, logEvent: any) => {
|
|
69
|
-
try {
|
|
70
|
-
const messages = [...logEvent.messages];
|
|
71
|
-
const severityNumber = mapLevelToSeverityNumber(levelLabel);
|
|
72
|
-
|
|
73
|
-
// Construct the message body
|
|
74
|
-
let body = '';
|
|
75
|
-
const msg = messages.pop();
|
|
76
|
-
|
|
77
|
-
if (typeof msg === 'string') {
|
|
78
|
-
body = msg;
|
|
79
|
-
} else if (msg) {
|
|
80
|
-
body = JSON.stringify(msg);
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
let category = 'spooky-client::unknown';
|
|
84
|
-
|
|
85
|
-
const attributes = {};
|
|
86
|
-
for (const msg of messages) {
|
|
87
|
-
if (typeof msg === 'object') {
|
|
88
|
-
if (msg.Category) {
|
|
89
|
-
category = msg.Category;
|
|
90
|
-
delete msg.Category;
|
|
91
|
-
}
|
|
92
|
-
Object.assign(attributes, msg);
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
// Emit to OTEL SDK
|
|
97
|
-
getOtelLogger(category).emit({
|
|
98
|
-
severityNumber: severityNumber,
|
|
99
|
-
severityText: levelLabel.toUpperCase(),
|
|
100
|
-
body: body,
|
|
101
|
-
attributes: {
|
|
102
|
-
...logEvent.bindings[0],
|
|
103
|
-
...attributes,
|
|
104
|
-
},
|
|
105
|
-
timestamp: new Date(logEvent.ts),
|
|
106
|
-
});
|
|
107
|
-
} catch (e) {
|
|
108
|
-
console.warn('Failed to transmit log to OTEL endpoint', e);
|
|
109
|
-
}
|
|
110
|
-
},
|
|
111
|
-
};
|
|
15
|
+
if (transmit) {
|
|
16
|
+
browserConfig.transmit = transmit;
|
|
112
17
|
}
|
|
113
18
|
|
|
114
19
|
return pino({
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { Logger } from 'pino';
|
|
2
|
-
import { PersistenceClient } from '../../types';
|
|
1
|
+
import type { Logger } from 'pino';
|
|
2
|
+
import type { PersistenceClient } from '../../types';
|
|
3
3
|
|
|
4
4
|
export class LocalStoragePersistenceClient implements PersistenceClient {
|
|
5
5
|
private logger: Logger;
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import type { Logger } from 'pino';
|
|
2
|
+
import type { PersistenceClient } from '../../types';
|
|
3
|
+
|
|
4
|
+
export class ResilientPersistenceClient implements PersistenceClient {
|
|
5
|
+
private logger: Logger;
|
|
6
|
+
|
|
7
|
+
constructor(
|
|
8
|
+
private inner: PersistenceClient,
|
|
9
|
+
logger: Logger
|
|
10
|
+
) {
|
|
11
|
+
this.logger = logger.child({ service: 'ResilientPersistenceClient' });
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
set<T>(key: string, value: T): Promise<void> {
|
|
15
|
+
return this.inner.set(key, value);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
async get<T>(key: string): Promise<T | null> {
|
|
19
|
+
try {
|
|
20
|
+
return await this.inner.get<T>(key);
|
|
21
|
+
} catch (e) {
|
|
22
|
+
this.logger.warn(
|
|
23
|
+
{ key, error: e, Category: 'sp00ky-client::ResilientPersistenceClient::get' },
|
|
24
|
+
'Persistence read failed, dropping key'
|
|
25
|
+
);
|
|
26
|
+
// Best-effort cleanup of the corrupt key; if removal also fails, the key
|
|
27
|
+
// just gets dropped again on the next failed read, so the failure is safe.
|
|
28
|
+
await this.inner.remove(key).catch((removeErr) => {
|
|
29
|
+
this.logger.debug(
|
|
30
|
+
{ key, error: removeErr, Category: 'sp00ky-client::ResilientPersistenceClient::get' },
|
|
31
|
+
'Failed to drop corrupt persistence key'
|
|
32
|
+
);
|
|
33
|
+
});
|
|
34
|
+
return null;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
remove(key: string): Promise<void> {
|
|
39
|
+
return this.inner.remove(key);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { PersistenceClient } from '../../types';
|
|
1
|
+
import type { PersistenceClient } from '../../types';
|
|
2
2
|
import { parseRecordIdString, surql } from '../../utils/index';
|
|
3
|
-
import { Logger } from 'pino';
|
|
4
|
-
import { AbstractDatabaseService } from '../database/database';
|
|
3
|
+
import type { Logger } from 'pino';
|
|
4
|
+
import type { AbstractDatabaseService } from '../database/database';
|
|
5
5
|
|
|
6
6
|
export class SurrealDBPersistenceClient implements PersistenceClient {
|
|
7
7
|
private logger: Logger;
|
|
@@ -15,11 +15,11 @@ export class SurrealDBPersistenceClient implements PersistenceClient {
|
|
|
15
15
|
|
|
16
16
|
async set<T>(key: string, val: T) {
|
|
17
17
|
try {
|
|
18
|
-
const id = parseRecordIdString(`
|
|
18
|
+
const id = parseRecordIdString(`_00_kv:${key}`);
|
|
19
19
|
await this.db.query(surql.seal(surql.upsert('id', 'data')), { id, data: { val } });
|
|
20
20
|
} catch (error) {
|
|
21
21
|
this.logger.error(
|
|
22
|
-
{ error, Category: '
|
|
22
|
+
{ error, Category: 'sp00ky-client::SurrealDBPersistenceClient::set' },
|
|
23
23
|
'Failed to set KV'
|
|
24
24
|
);
|
|
25
25
|
throw error;
|
|
@@ -28,7 +28,7 @@ export class SurrealDBPersistenceClient implements PersistenceClient {
|
|
|
28
28
|
|
|
29
29
|
async get<T>(key: string) {
|
|
30
30
|
try {
|
|
31
|
-
const id = parseRecordIdString(`
|
|
31
|
+
const id = parseRecordIdString(`_00_kv:${key}`);
|
|
32
32
|
const [result] = await this.db.query<[{ val: T }]>(
|
|
33
33
|
surql.seal(surql.selectById('id', ['val'])),
|
|
34
34
|
{
|
|
@@ -41,7 +41,7 @@ export class SurrealDBPersistenceClient implements PersistenceClient {
|
|
|
41
41
|
return result.val;
|
|
42
42
|
} catch (error) {
|
|
43
43
|
this.logger.warn(
|
|
44
|
-
{ error, Category: '
|
|
44
|
+
{ error, Category: 'sp00ky-client::SurrealDBPersistenceClient::get' },
|
|
45
45
|
'Failed to get KV'
|
|
46
46
|
);
|
|
47
47
|
return null;
|
|
@@ -50,11 +50,11 @@ export class SurrealDBPersistenceClient implements PersistenceClient {
|
|
|
50
50
|
|
|
51
51
|
async remove(key: string) {
|
|
52
52
|
try {
|
|
53
|
-
const id = parseRecordIdString(`
|
|
53
|
+
const id = parseRecordIdString(`_00_kv:${key}`);
|
|
54
54
|
await this.db.query(surql.seal(surql.delete('id')), { id });
|
|
55
55
|
} catch (err) {
|
|
56
56
|
this.logger.info(
|
|
57
|
-
{ err, Category: '
|
|
57
|
+
{ err, Category: 'sp00ky-client::SurrealDBPersistenceClient::remove' },
|
|
58
58
|
'Failed to delete KV'
|
|
59
59
|
);
|
|
60
60
|
}
|