@starkeep/storage-sqlite 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,101 @@
1
+ import { DatabaseAdapter, RawDatabase, Query, QueryResult, BatchOperation, Transaction, LabelUpsert, LabelRetraction, LabelValueReplacement, FindByLabelQuery, FindByLabelResult, StoredAvailability } from '@starkeep/storage-adapter';
2
+ import { DataRecord, StarkeepId, HLCTimestamp, MetadataRow, RecordLabel } from '@starkeep/protocol-primitives';
3
+
4
+ interface SqliteDatabaseAdapterOptions {
5
+ path: string | ":memory:";
6
+ /**
7
+ * How to open the connection, and how to close it.
8
+ *
9
+ * Injected so a second SQLite driver can back this adapter rather than
10
+ * duplicating it. Everything below this line already speaks only
11
+ * {@link RawDatabase} — `exec` and `prepare` — which is what item 10
12
+ * established; opening the connection was the single remaining line that
13
+ * named `node:sqlite`, and it is the one thing that genuinely cannot be
14
+ * driver-agnostic.
15
+ *
16
+ * **Required, and deliberately without a default.** It used to default to
17
+ * `node:sqlite`, which meant this module imported it statically — and a static
18
+ * `node:` import at module scope makes the whole package unbundleable on React
19
+ * Native, whatever driver the caller then passes. The Node driver now lives at
20
+ * `@starkeep/storage-sqlite/node`, so a Node caller writes one extra import
21
+ * and a handset can bundle this file at all.
22
+ *
23
+ * A default that only works on some platforms is worse than no default: it
24
+ * fails at bundle time, far from the line that chose it.
25
+ *
26
+ * React Native passes an op-sqlite driver; see `src/db/op-sqlite-driver.ts` in
27
+ * the `starkeep-apps` photos-mobile app for why that one emulates `prepare()`
28
+ * rather than using op-sqlite's prepared statements.
29
+ */
30
+ driver: SqliteDriver;
31
+ }
32
+ interface SqliteDriver {
33
+ open(path: string): RawDatabase;
34
+ /**
35
+ * Separate from the connection because `RawDatabase` deliberately does not
36
+ * carry `close()`. Consumers of a connection have no business closing it —
37
+ * only whoever opened it does, which is this adapter.
38
+ */
39
+ close(db: RawDatabase): void;
40
+ }
41
+ declare class SqliteDatabaseAdapter implements DatabaseAdapter {
42
+ private database;
43
+ private readonly options;
44
+ private readonly driver;
45
+ constructor(options: SqliteDatabaseAdapterOptions);
46
+ init(): Promise<void>;
47
+ close(): Promise<void>;
48
+ healthCheck(): Promise<boolean>;
49
+ private getDatabase;
50
+ /**
51
+ * The raw SQLite connection, so sibling subsystems (the sync engine's change
52
+ * log and state store, the resident set) can create side tables in the same
53
+ * database file. Only valid after `init()`.
54
+ *
55
+ * Typed as {@link RawDatabase} rather than `DatabaseSync`: returning the
56
+ * concrete Node type is what made a second driver impossible, since every
57
+ * consumer was then nailed to `node:sqlite`, which React Native does not
58
+ * have. `DatabaseSync` satisfies the narrower interface structurally, so
59
+ * nothing about this connection changes — only what callers may reach for.
60
+ */
61
+ getRawDatabase(): RawDatabase;
62
+ private runStmt;
63
+ private getRow;
64
+ private allRows;
65
+ put(record: DataRecord): Promise<void>;
66
+ get(id: StarkeepId): Promise<DataRecord | null>;
67
+ delete(id: StarkeepId, hlc: HLCTimestamp): Promise<void>;
68
+ getNodeWatermarks(): Promise<Record<string, HLCTimestamp>>;
69
+ query(query: Query): Promise<QueryResult>;
70
+ batch(operations: BatchOperation[]): Promise<void>;
71
+ transaction<T>(callback: (transaction: Transaction) => Promise<T>): Promise<T>;
72
+ putMetadata(typeId: string, row: MetadataRow): Promise<void>;
73
+ getMetadata(typeId: string, recordId: StarkeepId): Promise<MetadataRow | null>;
74
+ getMetadataByIds(typeId: string, recordIds: StarkeepId[]): Promise<Map<StarkeepId, MetadataRow>>;
75
+ deleteMetadata(typeId: string, recordId: StarkeepId): Promise<void>;
76
+ upsertLabels(labels: LabelUpsert[]): Promise<void>;
77
+ retractLabels(retractions: LabelRetraction[]): Promise<void>;
78
+ replaceLabelValues(replacements: LabelValueReplacement[]): Promise<void>;
79
+ getLabelsByRecordIds(recordIds: StarkeepId[]): Promise<Map<StarkeepId, RecordLabel[]>>;
80
+ findByLabel(query: FindByLabelQuery): Promise<FindByLabelResult>;
81
+ putLabel(label: RecordLabel): Promise<void>;
82
+ getLabel(recordId: StarkeepId, appId: string, key: string, value: string): Promise<RecordLabel | null>;
83
+ queryLabels(query: {
84
+ limit?: number;
85
+ cursor?: string;
86
+ }): Promise<{
87
+ labels: RecordLabel[];
88
+ nextCursor: string | null;
89
+ hasMore: boolean;
90
+ }>;
91
+ getLabelNodeWatermarks(): Promise<Record<string, HLCTimestamp>>;
92
+ getAvailability(objectStorageKeys: string[]): Promise<Map<string, StoredAvailability>>;
93
+ putAvailability(row: StoredAvailability): Promise<void>;
94
+ countRestoringObjects(): Promise<{
95
+ objectCount: number;
96
+ bytes: number;
97
+ }>;
98
+ tombstoneLabelsForRecord(recordId: StarkeepId, hlc: HLCTimestamp): Promise<void>;
99
+ }
100
+
101
+ export { SqliteDatabaseAdapter as S, type SqliteDatabaseAdapterOptions as a, type SqliteDriver as b };
@@ -0,0 +1,101 @@
1
+ import { DatabaseAdapter, RawDatabase, Query, QueryResult, BatchOperation, Transaction, LabelUpsert, LabelRetraction, LabelValueReplacement, FindByLabelQuery, FindByLabelResult, StoredAvailability } from '@starkeep/storage-adapter';
2
+ import { DataRecord, StarkeepId, HLCTimestamp, MetadataRow, RecordLabel } from '@starkeep/protocol-primitives';
3
+
4
+ interface SqliteDatabaseAdapterOptions {
5
+ path: string | ":memory:";
6
+ /**
7
+ * How to open the connection, and how to close it.
8
+ *
9
+ * Injected so a second SQLite driver can back this adapter rather than
10
+ * duplicating it. Everything below this line already speaks only
11
+ * {@link RawDatabase} — `exec` and `prepare` — which is what item 10
12
+ * established; opening the connection was the single remaining line that
13
+ * named `node:sqlite`, and it is the one thing that genuinely cannot be
14
+ * driver-agnostic.
15
+ *
16
+ * **Required, and deliberately without a default.** It used to default to
17
+ * `node:sqlite`, which meant this module imported it statically — and a static
18
+ * `node:` import at module scope makes the whole package unbundleable on React
19
+ * Native, whatever driver the caller then passes. The Node driver now lives at
20
+ * `@starkeep/storage-sqlite/node`, so a Node caller writes one extra import
21
+ * and a handset can bundle this file at all.
22
+ *
23
+ * A default that only works on some platforms is worse than no default: it
24
+ * fails at bundle time, far from the line that chose it.
25
+ *
26
+ * React Native passes an op-sqlite driver; see `src/db/op-sqlite-driver.ts` in
27
+ * the `starkeep-apps` photos-mobile app for why that one emulates `prepare()`
28
+ * rather than using op-sqlite's prepared statements.
29
+ */
30
+ driver: SqliteDriver;
31
+ }
32
+ interface SqliteDriver {
33
+ open(path: string): RawDatabase;
34
+ /**
35
+ * Separate from the connection because `RawDatabase` deliberately does not
36
+ * carry `close()`. Consumers of a connection have no business closing it —
37
+ * only whoever opened it does, which is this adapter.
38
+ */
39
+ close(db: RawDatabase): void;
40
+ }
41
+ declare class SqliteDatabaseAdapter implements DatabaseAdapter {
42
+ private database;
43
+ private readonly options;
44
+ private readonly driver;
45
+ constructor(options: SqliteDatabaseAdapterOptions);
46
+ init(): Promise<void>;
47
+ close(): Promise<void>;
48
+ healthCheck(): Promise<boolean>;
49
+ private getDatabase;
50
+ /**
51
+ * The raw SQLite connection, so sibling subsystems (the sync engine's change
52
+ * log and state store, the resident set) can create side tables in the same
53
+ * database file. Only valid after `init()`.
54
+ *
55
+ * Typed as {@link RawDatabase} rather than `DatabaseSync`: returning the
56
+ * concrete Node type is what made a second driver impossible, since every
57
+ * consumer was then nailed to `node:sqlite`, which React Native does not
58
+ * have. `DatabaseSync` satisfies the narrower interface structurally, so
59
+ * nothing about this connection changes — only what callers may reach for.
60
+ */
61
+ getRawDatabase(): RawDatabase;
62
+ private runStmt;
63
+ private getRow;
64
+ private allRows;
65
+ put(record: DataRecord): Promise<void>;
66
+ get(id: StarkeepId): Promise<DataRecord | null>;
67
+ delete(id: StarkeepId, hlc: HLCTimestamp): Promise<void>;
68
+ getNodeWatermarks(): Promise<Record<string, HLCTimestamp>>;
69
+ query(query: Query): Promise<QueryResult>;
70
+ batch(operations: BatchOperation[]): Promise<void>;
71
+ transaction<T>(callback: (transaction: Transaction) => Promise<T>): Promise<T>;
72
+ putMetadata(typeId: string, row: MetadataRow): Promise<void>;
73
+ getMetadata(typeId: string, recordId: StarkeepId): Promise<MetadataRow | null>;
74
+ getMetadataByIds(typeId: string, recordIds: StarkeepId[]): Promise<Map<StarkeepId, MetadataRow>>;
75
+ deleteMetadata(typeId: string, recordId: StarkeepId): Promise<void>;
76
+ upsertLabels(labels: LabelUpsert[]): Promise<void>;
77
+ retractLabels(retractions: LabelRetraction[]): Promise<void>;
78
+ replaceLabelValues(replacements: LabelValueReplacement[]): Promise<void>;
79
+ getLabelsByRecordIds(recordIds: StarkeepId[]): Promise<Map<StarkeepId, RecordLabel[]>>;
80
+ findByLabel(query: FindByLabelQuery): Promise<FindByLabelResult>;
81
+ putLabel(label: RecordLabel): Promise<void>;
82
+ getLabel(recordId: StarkeepId, appId: string, key: string, value: string): Promise<RecordLabel | null>;
83
+ queryLabels(query: {
84
+ limit?: number;
85
+ cursor?: string;
86
+ }): Promise<{
87
+ labels: RecordLabel[];
88
+ nextCursor: string | null;
89
+ hasMore: boolean;
90
+ }>;
91
+ getLabelNodeWatermarks(): Promise<Record<string, HLCTimestamp>>;
92
+ getAvailability(objectStorageKeys: string[]): Promise<Map<string, StoredAvailability>>;
93
+ putAvailability(row: StoredAvailability): Promise<void>;
94
+ countRestoringObjects(): Promise<{
95
+ objectCount: number;
96
+ bytes: number;
97
+ }>;
98
+ tombstoneLabelsForRecord(recordId: StarkeepId, hlc: HLCTimestamp): Promise<void>;
99
+ }
100
+
101
+ export { SqliteDatabaseAdapter as S, type SqliteDatabaseAdapterOptions as a, type SqliteDriver as b };
package/dist/index.cjs CHANGED
@@ -34,9 +34,6 @@ __export(index_exports, {
34
34
  module.exports = __toCommonJS(index_exports);
35
35
 
36
36
  // src/adapter.ts
37
- var import_node_sqlite = require("sqlite");
38
- var import_node_fs = require("fs");
39
- var import_node_path = require("path");
40
37
  var import_protocol_primitives3 = require("@starkeep/protocol-primitives");
41
38
  var import_storage_adapter2 = require("@starkeep/storage-adapter");
42
39
  var import_kysely3 = require("kysely");
@@ -119,6 +116,16 @@ function buildSelectQuery(query) {
119
116
  qb = applyFilter(qb, filter);
120
117
  }
121
118
  }
119
+ if (query.excludeLabel) {
120
+ const { appId, key } = query.excludeLabel;
121
+ qb = qb.where(
122
+ (eb) => eb.not(
123
+ eb.exists(
124
+ eb.selectFrom("shared_record_labels").select("record_id").whereRef("shared_record_labels.record_id", "=", "shared_records.id").where("app_id", "=", appId).where("key", "=", key).where("deleted_at", "is", null)
125
+ )
126
+ )
127
+ );
128
+ }
122
129
  if (query.cursor) {
123
130
  qb = qb.where("id", ">", query.cursor);
124
131
  }
@@ -177,6 +184,15 @@ function applyLocalSchemaDdl(db) {
177
184
  compiler.schema.createIndex("idx_shared_records_node_watermark").ifNotExists().on("shared_records").columns(["node_id", "updated_at"]),
178
185
  compiler.schema.createIndex("idx_shared_records_origin_app").ifNotExists().on("shared_records").column("origin_app_id"),
179
186
  compiler.schema.createIndex("idx_shared_records_parent_id").ifNotExists().on("shared_records").column("parent_id"),
187
+ // Record-level dedup: "is there already a live record for these bytes
188
+ // under this parent?", asked on **every** registration.
189
+ //
190
+ // The (filename, content_hash) unique index below cannot serve it — its
191
+ // leading column is the filename, so a hash-only lookup would scan the
192
+ // table once per write. Leading with content_hash is what makes the dedup
193
+ // check affordable enough to be unconditional, and unconditional is what
194
+ // makes "one object key, one record" an invariant rather than a hope.
195
+ compiler.schema.createIndex("idx_shared_records_content_hash").ifNotExists().on("shared_records").columns(["content_hash", "parent_id", "deleted_at"]),
180
196
  // Duplicate-file prevention: (filename + bytes) is unique among live
181
197
  // records. Tombstoned rows (deleted_at IS NOT NULL) are excluded so a
182
198
  // re-upload after delete is allowed. Records with NULL filename are not
@@ -213,6 +229,9 @@ function applyLocalSchemaDdl(db) {
213
229
  for (const index of sharedRecordLabelsIndexes) {
214
230
  db.exec(index.compile().sql);
215
231
  }
232
+ db.exec(
233
+ compiler.schema.createTable("shared_object_availability").ifNotExists().addColumn("object_storage_key", "text", (c) => c.primaryKey()).addColumn("state", "text", (c) => c.notNull()).addColumn("tier", "text").addColumn("expected_latency_hours", "integer").addColumn("ready_at_ms", "integer").addColumn("restored_until_ms", "integer").addColumn("observed_at_ms", "integer", (c) => c.notNull()).compile().sql
234
+ );
216
235
  db.exec(
217
236
  compiler.schema.createTable("shared_app_label_keys").ifNotExists().addColumn("app_id", "text", (c) => c.notNull()).addColumn("key", "text", (c) => c.notNull()).addColumn("description", "text").addColumn("created_at", "text", (c) => c.notNull().defaultTo(import_kysely2.sql`(datetime('now'))`)).addPrimaryKeyConstraint("pk_shared_app_label_keys", ["app_id", "key"]).compile().sql
218
237
  );
@@ -254,28 +273,36 @@ function initializeLocalSchema(db) {
254
273
  }
255
274
 
256
275
  // src/adapter.ts
276
+ var AVAILABILITY = "shared_object_availability";
277
+ function rowToAvailability(row) {
278
+ return {
279
+ objectStorageKey: row.object_storage_key,
280
+ state: row.state,
281
+ tier: row.tier,
282
+ expectedLatencyHours: row.expected_latency_hours,
283
+ readyAtMs: row.ready_at_ms,
284
+ restoredUntilMs: row.restored_until_ms,
285
+ observedAtMs: row.observed_at_ms
286
+ };
287
+ }
257
288
  var LABELS = {
258
289
  table: "shared_record_labels"
259
290
  };
260
291
  var SqliteDatabaseAdapter = class {
261
292
  database = null;
262
293
  options;
294
+ driver;
263
295
  constructor(options) {
264
296
  this.options = options;
297
+ this.driver = options.driver;
265
298
  }
266
299
  async init() {
267
300
  if (this.database) return;
268
- if (this.options.path !== ":memory:") {
269
- const dir = (0, import_node_path.dirname)(this.options.path);
270
- if (!(0, import_node_fs.existsSync)(dir)) {
271
- (0, import_node_fs.mkdirSync)(dir, { recursive: true });
272
- }
273
- }
274
- this.database = new import_node_sqlite.DatabaseSync(this.options.path);
301
+ this.database = this.driver.open(this.options.path);
275
302
  initializeLocalSchema(this.database);
276
303
  }
277
304
  async close() {
278
- this.database?.close();
305
+ if (this.database) this.driver.close(this.database);
279
306
  this.database = null;
280
307
  }
281
308
  async healthCheck() {
@@ -292,25 +319,31 @@ var SqliteDatabaseAdapter = class {
292
319
  return this.database;
293
320
  }
294
321
  /**
295
- * Returns the raw SQLite connection so sibling subsystems (e.g. the sync
296
- * engine's change log + state store) can create side tables in the same
297
- * database file. Callers must only use this after `init()`.
322
+ * The raw SQLite connection, so sibling subsystems (the sync engine's change
323
+ * log and state store, the resident set) can create side tables in the same
324
+ * database file. Only valid after `init()`.
325
+ *
326
+ * Typed as {@link RawDatabase} rather than `DatabaseSync`: returning the
327
+ * concrete Node type is what made a second driver impossible, since every
328
+ * consumer was then nailed to `node:sqlite`, which React Native does not
329
+ * have. `DatabaseSync` satisfies the narrower interface structurally, so
330
+ * nothing about this connection changes — only what callers may reach for.
298
331
  */
299
332
  getRawDatabase() {
300
333
  return this.getDatabase();
301
334
  }
335
+ // The casts these three used to carry named `DatabaseSync["prepare"]` to
336
+ // borrow node:sqlite's parameter types. `RawStatement` declares `unknown[]`
337
+ // directly, so they were already no-ops — and they were the last thing in
338
+ // this file referring to a concrete driver.
302
339
  runStmt(sql3, ...params) {
303
340
  this.getDatabase().prepare(sql3).run(...params);
304
341
  }
305
342
  getRow(sql3, ...params) {
306
- return this.getDatabase().prepare(sql3).get(
307
- ...params
308
- );
343
+ return this.getDatabase().prepare(sql3).get(...params);
309
344
  }
310
345
  allRows(sql3, ...params) {
311
- return this.getDatabase().prepare(sql3).all(
312
- ...params
313
- );
346
+ return this.getDatabase().prepare(sql3).all(...params);
314
347
  }
315
348
  async put(record) {
316
349
  const row = recordToRow(record);
@@ -509,6 +542,47 @@ var SqliteDatabaseAdapter = class {
509
542
  for (const row of rows) out[row.node_id] = (0, import_protocol_primitives3.deserializeHLC)(row.max_updated_at);
510
543
  return out;
511
544
  }
545
+ // ---- Object availability ------------------------------------------------
546
+ async getAvailability(objectStorageKeys) {
547
+ const out = /* @__PURE__ */ new Map();
548
+ if (objectStorageKeys.length === 0) return out;
549
+ const compiled = compiler.selectFrom(AVAILABILITY).selectAll().where("object_storage_key", "in", objectStorageKeys).compile();
550
+ const rows = this.allRows(compiled.sql, ...compiled.parameters);
551
+ for (const row of rows) out.set(row.object_storage_key, rowToAvailability(row));
552
+ return out;
553
+ }
554
+ async putAvailability(row) {
555
+ const compiled = compiler.insertInto(AVAILABILITY).values({
556
+ object_storage_key: row.objectStorageKey,
557
+ state: row.state,
558
+ tier: row.tier,
559
+ expected_latency_hours: row.expectedLatencyHours,
560
+ ready_at_ms: row.readyAtMs,
561
+ restored_until_ms: row.restoredUntilMs,
562
+ observed_at_ms: row.observedAtMs
563
+ }).onConflict(
564
+ (oc) => oc.column("object_storage_key").doUpdateSet((eb) => ({
565
+ state: eb.ref("excluded.state"),
566
+ tier: eb.ref("excluded.tier"),
567
+ expected_latency_hours: eb.ref("excluded.expected_latency_hours"),
568
+ ready_at_ms: eb.ref("excluded.ready_at_ms"),
569
+ restored_until_ms: eb.ref("excluded.restored_until_ms"),
570
+ observed_at_ms: eb.ref("excluded.observed_at_ms")
571
+ }))
572
+ ).compile();
573
+ this.runStmt(compiled.sql, ...compiled.parameters);
574
+ }
575
+ async countRestoringObjects() {
576
+ const compiled = compiler.selectFrom(AVAILABILITY).innerJoin("shared_records", "shared_records.object_storage_key", `${AVAILABILITY}.object_storage_key`).select(({ fn }) => [
577
+ fn.count(`${AVAILABILITY}.object_storage_key`).as("object_count"),
578
+ fn.sum("shared_records.size_bytes").as("bytes")
579
+ ]).where(`${AVAILABILITY}.state`, "=", import_kysely3.sql.lit("restoring")).compile();
580
+ const row = this.getRow(
581
+ compiled.sql,
582
+ ...compiled.parameters
583
+ );
584
+ return { objectCount: row?.object_count ?? 0, bytes: row?.bytes ?? 0 };
585
+ }
512
586
  async tombstoneLabelsForRecord(recordId, hlc) {
513
587
  const query = (0, import_storage_adapter2.buildTombstoneLabelsForRecord)(compiler, LABELS, recordId, hlc);
514
588
  this.runStmt(query.sql, ...query.parameters);