@chipmobilesdk/rn-local-db 0.3.0 → 0.4.0

package/src/adapter/schema-sync.ts

@@ -0,0 +1,155 @@
+/**
+ * Bringing the physical tables in line with the declared schema, and skipping
+ * that work when nothing changed (LDB-07, LDB-11).
+ *
+ * Until 0.4.0 every open ran the whole set — CREATE TABLE, PRAGMA table_info,
+ * CREATE INDEX per index, for every collection — some sixty native round-trips
+ * for ten collections, on the startup path. The applied schema is now recorded
+ * in `_meta`; an open whose declaration matches it runs no DDL at all.
+ */
+import type { DatabaseConfig } from '../contract/config';
+import type { CollectionSchema } from '../contract/schema';
+import { ensureAttachmentTable } from '../artifacts/store';
+import type { Connection } from '../sqlite/connection';
+import {
+	addColumnSql,
+	createIndexSql,
+	createTableSql,
+	physicalIndexName,
+} from '../sqlite/row-codec';
+import { text } from '../sqlite/values';
+
+/** Bumped whenever the DDL this package generates changes, so it is re-applied once. */
+const DDL_VERSION = 1;
+const FINGERPRINT_KEY = 'schema_fingerprint';
+
+/** What was declared, and by which generation of this package's DDL. */
+function fingerprintOf(config: DatabaseConfig): string {
+	return JSON.stringify([DDL_VERSION, config.collections]);
+}
+
+async function appliedFingerprint(connection: Connection): Promise<string | null> {
+	try {
+		const result = await connection.execute('SELECT value FROM _meta WHERE key = ?', [
+			FINGERPRINT_KEY,
+		]);
+		const row = result.rows[0];
+		return row ? text(row.value) : null;
+	} catch {
+		// A database that has never been opened has no `_meta` yet.
+		return null;
+	}
+}
+
+/** `unique` and the column list of an index, read back from the statement SQLite stored. */
+function definitionOf(sql: string): string {
+	const unique = /^\s*CREATE\s+UNIQUE\s/i.test(sql);
+	const columns = sql.slice(sql.lastIndexOf('(') + 1, sql.lastIndexOf(')')).replace(/["\s]/g, '');
+	return `${unique ? 'unique' : 'plain'}:${columns}`;
+}
+
+/**
+ * Indexes follow their declaration (LDB-11). `CREATE INDEX IF NOT EXISTS` keyed
+ * on the name alone meant that adding `unique: true` to an existing index, or
+ * changing its fields, did nothing, and an index no longer declared stayed
+ * forever. An index whose definition changed is dropped and recreated, and one
+ * of this package's that is no longer declared is dropped. Only indexes carrying
+ * the collection's own prefix are touched.
+ */
+async function syncIndexes(connection: Connection, schema: CollectionSchema): Promise<void> {
+	const table = `c_${schema.name}`;
+	const existing = await connection.execute(
+		"SELECT name, sql FROM sqlite_master WHERE type = 'index' AND tbl_name = ? AND sql IS NOT NULL",
+		[table],
+	);
+	const current = new Map(existing.rows.map(row => [text(row.name), text(row.sql)]));
+	const wanted = new Map(
+		createIndexSql(schema).map(statement => {
+			const name = /INDEX IF NOT EXISTS "([^"]+)"/.exec(statement)?.[1] ?? '';
+			return [name, statement] as const;
+		}),
+	);
+
+	for (const index of schema.indexes ?? []) {
+		// Indexes created before 0.2.0 carry the bare declared name; they are
+		// recreated under the collection-scoped one. No row data moves.
+		if (current.has(index.name) && index.name !== physicalIndexName(schema, index.name)) {
+			await connection.execute(`DROP INDEX IF EXISTS "${index.name}"`);
+		}
+	}
+	for (const [name, sql] of current) {
+		const statement = wanted.get(name);
+		const ours = name.startsWith(`${table}__`);
+		if (
+			(statement === undefined && ours) ||
+			(statement && definitionOf(statement) !== definitionOf(sql))
+		) {
+			await connection.execute(`DROP INDEX IF EXISTS "${name}"`);
+		}
+	}
+	for (const statement of wanted.values()) {
+		await connection.execute(statement);
+	}
+}
+
+/**
+ * Adding a column rather than recreating the table is what makes declaring a
+ * new queryable field non-destructive: the payload already holds the value, so
+ * the new column is backfilled from it and no record is rewritten by hand
+ * (research.md R6).
+ */
+async function ensureCollection(connection: Connection, schema: CollectionSchema): Promise<void> {
+	await connection.execute(createTableSql(schema));
+
+	const info = await connection.execute(`PRAGMA table_info("c_${schema.name}")`);
+	const present = new Set(info.rows.map(row => text(row.name)));
+
+	for (const field of schema.fields ?? []) {
+		const column = `f_${field.name}`;
+		if (present.has(column)) {
+			continue;
+		}
+		await connection.execute(addColumnSql(schema, field));
+		// The same projection a write applies (`projectValue`): an object or an
+		// array projects to NULL. A bare json_extract backfilled those as JSON
+		// text, so a filter matched old records and new ones differently (LDB-11).
+		const path = `'$."${field.name}"'`;
+		await connection.execute(
+			`UPDATE "c_${schema.name}" SET "${column}" = CASE WHEN json_type(payload, ${path}) IN ('object', 'array') THEN NULL ELSE json_extract(payload, ${path}) END`,
+		);
+	}
+
+	await syncIndexes(connection, schema);
+}
+
+export interface SchemaSync {
+	/** True when the stored schema already matched, so `_meta` exists and no DDL ran. */
+	readonly unchanged: boolean;
+	/** Records the declaration as applied. Call once the open has succeeded. */
+	record(): Promise<void>;
+}
+
+export async function syncSchema(
+	connection: Connection,
+	config: DatabaseConfig,
+): Promise<SchemaSync> {
+	const fingerprint = fingerprintOf(config);
+	if ((await appliedFingerprint(connection)) === fingerprint) {
+		return { unchanged: true, record: async () => undefined };
+	}
+
+	for (const schema of config.collections) {
+		await ensureCollection(connection, schema);
+	}
+	await ensureAttachmentTable(connection);
+
+	return {
+		unchanged: false,
+		async record() {
+			await connection.execute(
+				'INSERT INTO _meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value',
+				[FINGERPRINT_KEY, fingerprint],
+			);
+		},
+	};
+}

package/src/artifacts/cascade.ts

@@ -19,7 +19,6 @@ import type { Executor } from '../sqlite/connection';
 import { artifactPath } from './paths';
 import {
 	deleteReference,
-	listReferences,
 	listReferencesFor,
 	type ArtifactContext,
 	type DeferredUnlinks,
@@ -99,16 +98,3 @@ export async function cascadeForOwner(
 		deferred,
 	);
 }
-
-/** Everything in the scope, for scope deletion. */
-export async function cascadeAll(
-	context: ArtifactContext,
-	executor: Executor,
-): Promise<DeletionOutcome> {
-	const references = await listReferences(executor);
-	return removeAll(
-		context,
-		executor,
-		references.map(reference => reference.id),
-	);
-}

package/src/artifacts/store.ts

@@ -152,6 +152,12 @@ async function writePlaintext(path: string, content: string): Promise<void> {
 	await fileSystem().writeFile(path, content, 'base64');
 }
 
+/** Where an artifact's bytes come from: how many, and any one chunk of them. */
+type ByteSource = {
+	size: number;
+	chunk(offset: number, length: number): Promise<Uint8Array>;
+};
+
 /**
  * Encrypted write, one chunk in flight.
  *
@@ -159,43 +165,87 @@ async function writePlaintext(path: string, content: string): Promise<void> {
  * plaintext length — are all known from the input length before any
  * encryption happens. Only the tags depend on the ciphertext, and they follow
  * their own chunk, so the whole write is append-only and peak memory is one
- * chunk regardless of artifact size (FR-107a).
+ * chunk regardless of artifact size when the source is a file (FR-107a).
  */
 async function writeEncrypted(
 	path: string,
 	key: Uint8Array,
-	plaintext: Uint8Array,
+	source: ByteSource,
 ): Promise<{ chunkSize: number; chunkCount: number }> {
-	const header = headerFor(plaintext.length, Budgets.ARTIFACT_CHUNK_BYTES);
+	const header = headerFor(source.size, Budgets.ARTIFACT_CHUNK_BYTES);
 	await fileSystem().writeFile(path, bytesToBase64(encodeHeader(header)), 'base64');
 
 	for (let index = 0; index < header.chunkCount; index += 1) {
-		const start = index * header.chunkSize;
-		const chunk = plaintext.subarray(start, start + chunkPlaintextLength(header, index));
+		const chunk = await source.chunk(index * header.chunkSize, chunkPlaintextLength(header, index));
 		await fileSystem().appendFile(path, bytesToBase64(encryptChunk(key, index, chunk)), 'base64');
 	}
 
 	return { chunkSize: header.chunkSize, chunkCount: header.chunkCount };
 }
 
+function sourceFileOf(input: AttachmentInput): string | null {
+	if (typeof input.sourcePath !== 'string') {
+		return null;
+	}
+	return input.sourcePath.startsWith('file://')
+		? input.sourcePath.slice('file://'.length)
+		: input.sourcePath;
+}
+
+/**
+ * The input as something to read from. A base64 `content` is decoded whole,
+ * which is the cost of that form; a `sourcePath` is only measured here and
+ * read a chunk at a time, or not read by JS at all (LDB-09).
+ */
+async function byteSourceFor(input: AttachmentInput): Promise<ByteSource> {
+	const file = sourceFileOf(input);
+	if (file === null) {
+		if (typeof input.content !== 'string') {
+			raise(DiagnosticCodes.CONFIG_INVALID, {
+				message: 'an attachment needs content or sourcePath',
+				detail: { attachmentId: input.id },
+			});
+		}
+		const plaintext = base64ToBytes(input.content);
+		return {
+			size: plaintext.length,
+			chunk: async (offset, length) => plaintext.subarray(offset, offset + length),
+		};
+	}
+	let size: number;
+	try {
+		size = (await fileSystem().stat(file)).size;
+	} catch (error) {
+		raiseNative(error, { operation: 'attachment' });
+	}
+	return {
+		size,
+		chunk: async (offset, length) =>
+			base64ToBytes(await fileSystem().read(file, length, offset, 'base64')),
+	};
+}
+
 export async function writeArtifact(
 	context: ArtifactContext,
 	input: AttachmentInput,
 ): Promise<AttachmentReference> {
 	throwIfFaulted('attachmentWrite');
-	const plaintext = base64ToBytes(input.content);
-	assertArtifactWithinBudget(input.id, plaintext.length);
+	const source = await byteSourceFor(input);
+	assertArtifactWithinBudget(input.id, source.size);
 
 	const path = artifactPath(context.layout, context.scopeKey, input.id);
 	const key = context.artifactKey(input.id);
+	const file = sourceFileOf(input);
 
 	let chunking: { chunkSize: number; chunkCount: number } | undefined;
 	try {
 		await fileSystem().mkdir(artifactsDirectory(context.layout, context.scopeKey));
 		if (key) {
-			chunking = await writeEncrypted(path, key, plaintext);
+			chunking = await writeEncrypted(path, key, source);
+		} else if (file !== null) {
+			await fileSystem().copyFile(file, path);
 		} else {
-			await writePlaintext(path, input.content);
+			await writePlaintext(path, input.content as string);
 		}
 	} catch (error) {
 		// A partial file with no reference is an orphan reconciliation can
@@ -207,7 +257,7 @@ export async function writeArtifact(
 	const reference: AttachmentReference = {
 		id: input.id,
 		contentType: input.contentType,
-		size: plaintext.length,
+		size: source.size,
 		createdAt: context.now(),
 	};
 	if (input.owner) {

package/src/contract/attachment.ts

@@ -11,15 +11,36 @@ export type AttachmentOwner = {
 	id: string;
 };
 
-export type AttachmentInput = {
+type AttachmentDescription = {
 	id: string;
 	/** Absent means the attachment belongs to the scope rather than a record. */
 	owner?: AttachmentOwner | undefined;
 	contentType: string;
-	/** Base64 content. Written incrementally; never held whole when encrypted. */
-	content: string;
 };
 
+/**
+ * Where the bytes come from. Exactly one of the two.
+ *
+ * `sourcePath` is the one to use for anything large — a photo, a recording.
+ * The file is copied (or, when encrypted, read and encrypted 256 KB at a
+ * time), so memory does not grow with its size. `content` holds the whole
+ * payload in JS as base64 and is decoded in one piece: fine for a thumbnail,
+ * several times the file's size in memory for a video.
+ */
+export type AttachmentInput = AttachmentDescription &
+	(
+		| {
+				/** The complete content, base64. Held and decoded whole. */
+				content: string;
+				sourcePath?: undefined;
+		  }
+		| {
+				/** Absolute path of a file the app can read. A `file://` prefix is accepted. */
+				sourcePath: string;
+				content?: undefined;
+		  }
+	);
+
 export type AttachmentReference = {
 	id: string;
 	owner?: AttachmentOwner | undefined;

package/src/contract/migration.ts

@@ -10,9 +10,12 @@ import type { Transaction } from './transaction';
 /** Low-level escape hatch for schema work a migration needs. */
 // Attachments are not part of a migration: a migration reshapes records, and
 // files written inside one would survive its rollback with nothing to own them.
+/** A value SQLite can bind. */
+export type SqlParameter = string | number | boolean | null;
+
 export type MigrationContext = Omit<Transaction, 'attachments'> & {
 	/** Execute a raw statement against the scope's database. */
-	execute(sql: string, params?: unknown[]): Promise<void>;
+	execute(sql: string, params?: SqlParameter[]): Promise<void>;
 };
 
 export type Migration = {

package/src/diagnostics/dev-log.ts

@@ -73,7 +73,7 @@ export function isLogging(): boolean {
  * installed and the build is a development build, so a production bundle pays
  * nothing and emits nothing (FR-081).
  */
-export function instrument<T extends Record<string, (...args: never[]) => unknown>>(
+export function instrument<T extends object>(
 	collection: string,
 	adapter: 'sqlite' | 'memory',
 	operations: T,
@@ -85,6 +85,10 @@ export function instrument<T extends Record<string, (...args: never[]) => unknow
 
 	const wrapped: Record<string, unknown> = {};
 	for (const [name, operation] of Object.entries(operations)) {
+		if (typeof operation !== 'function') {
+			wrapped[name] = operation;
+			continue;
+		}
 		wrapped[name] = async (...args: never[]): Promise<unknown> => {
 			const started = Date.now();
 			try {

package/src/migration/executor.ts

@@ -21,7 +21,7 @@
 
 import type { DatabaseConfig } from '../contract/config';
 import { DiagnosticCodes } from '../contract/errors';
-import type { MigrationContext } from '../contract/migration';
+import type { MigrationContext, SqlParameter } from '../contract/migration';
 import type { StorableRecordData } from '../contract/record';
 import type { CollectionSchema } from '../contract/schema';
 import type { CollectionOps } from '../contract/transaction';
@@ -29,7 +29,13 @@ import { raise } from '../diagnostics/errors';
 import { throwIfFaulted } from '../diagnostics/faults';
 import { createSqlCollection } from '../adapter/collection';
 import type { Connection, Executor } from '../sqlite/connection';
-import { ensureLedger, readSchemaVersion, recordAttempt, writeSchemaVersion } from './ledger';
+import {
+	ensureLedger,
+	FORMAT_VERSION,
+	readVersions,
+	recordAttempt,
+	writeSchemaVersion,
+} from './ledger';
 import { pendingMigrations, validateMigrationChain } from './validate-chain';
 
 export type MigrationRunContext = {
@@ -44,6 +50,8 @@ export type MigrationRunContext = {
 	 * still be holding something usable (FR-050).
 	 */
 	closeOnFailure(): Promise<void>;
+	/** The bookkeeping tables are known to exist, so creating them is skipped (LDB-07). */
+	ledgerReady?: boolean;
 };
 
 function migrationContextFor(context: MigrationRunContext, executor: Executor): MigrationContext {
@@ -66,8 +74,8 @@ function migrationContextFor(context: MigrationRunContext, executor: Executor):
 				message: 'a migration cannot abort its own transaction',
 			});
 		},
-		async execute(sql: string, params?: unknown[]): Promise<void> {
-			await executor.execute(sql, params as never);
+		async execute(sql: string, params?: SqlParameter[]): Promise<void> {
+			await executor.execute(sql, params);
 		},
 	};
 }
@@ -75,9 +83,22 @@ function migrationContextFor(context: MigrationRunContext, executor: Executor):
 export async function runMigrations(context: MigrationRunContext): Promise<number> {
 	const { config, connection } = context;
 	validateMigrationChain(config);
-	await ensureLedger(connection);
+	// Skipped when the open already found its bookkeeping tables in place.
+	if (context.ledgerReady !== true) {
+		await ensureLedger(connection);
+	}
+
+	const { schema: recorded, format } = await readVersions(connection);
 
-	const recorded = await readSchemaVersion(connection);
+	if (format > FORMAT_VERSION) {
+		// A file laid out by a newer release of this package. Reading it with
+		// older code is the same hazard as a schema downgrade, one layer down.
+		await context.closeOnFailure();
+		raise(DiagnosticCodes.SCHEMA_DOWNGRADE, {
+			message: 'database was written by a newer version of this package',
+			detail: { recordedVersion: format, version: FORMAT_VERSION },
+		});
+	}
 
 	if (recorded > config.schemaVersion) {
 		// Opening anyway would mean newer data read by older code, which drops

package/src/migration/ledger.ts

@@ -12,9 +12,9 @@
  * with a long half-life, sitting in a file the next crash report might carry.
  */
 
-import type { MigrationLedgerEntry, MigrationOutcome } from '../contract/migration';
+import type { MigrationOutcome } from '../contract/migration';
 import type { Executor } from '../sqlite/connection';
-import { maybeText, num, text } from '../sqlite/values';
+import { text } from '../sqlite/values';
 
 /** Physical layout version, independent of the app's schema version. */
 export const FORMAT_VERSION = 1;
@@ -44,20 +44,23 @@ export async function ensureLedger(executor: Executor): Promise<void> {
 	);
 }
 
-async function readMeta(executor: Executor, key: string): Promise<string | null> {
-	const result = await executor.execute('SELECT value FROM _meta WHERE key = ?', [key]);
-	const row = result.rows[0];
-	return row ? text(row.value) : null;
-}
-
-export async function readSchemaVersion(executor: Executor): Promise<number> {
-	const raw = await readMeta(executor, 'schema_version');
-	return raw === null ? 0 : Number(raw);
-}
+export type RecordedVersions = {
+	/** The app's schema version this scope has been migrated to. */
+	schema: number;
+	/** The physical layout this package wrote the file with. */
+	format: number;
+};
 
-export async function readFormatVersion(executor: Executor): Promise<number> {
-	const raw = await readMeta(executor, 'format_version');
-	return raw === null ? FORMAT_VERSION : Number(raw);
+/** Both versions in one read: every open asks for them. */
+export async function readVersions(executor: Executor): Promise<RecordedVersions> {
+	const result = await executor.execute(
+		"SELECT key, value FROM _meta WHERE key IN ('schema_version', 'format_version')",
+	);
+	const values = new Map(result.rows.map(row => [text(row.key), Number(text(row.value))]));
+	return {
+		schema: values.get('schema_version') ?? 0,
+		format: values.get('format_version') ?? FORMAT_VERSION,
+	};
 }
 
 export async function writeSchemaVersion(executor: Executor, version: number): Promise<void> {
@@ -81,15 +84,3 @@ export async function recordAttempt(
 		[entry.version, entry.attemptedAt, entry.outcome, entry.failureCode ?? null],
 	);
 }
-
-export async function readLedger(executor: Executor): Promise<MigrationLedgerEntry[]> {
-	const result = await executor.execute(
-		'SELECT version, attempted_at, outcome, failure_code FROM _migrations ORDER BY version ASC',
-	);
-	return result.rows.map(row => ({
-		version: num(row.version),
-		attemptedAt: num(row.attempted_at),
-		outcome: text(row.outcome) as MigrationOutcome,
-		...(maybeText(row.failure_code) === null ? {} : { failureCode: text(row.failure_code) }),
-	}));
-}

package/src/platform/fs.ts

@@ -40,6 +40,8 @@ export type FileSystem = {
 	readFile(path: string, encoding?: string): Promise<string>;
 	/** Chunked read at an offset — what FR-107a's flat memory bound rests on. */
 	read(path: string, length: number, position: number, encoding?: string): Promise<string>;
+	/** One native call, no bytes through JS: how an unencrypted file becomes an artifact. */
+	copyFile(from: string, into: string): Promise<void>;
 	stat(path: string): Promise<FileStat>;
 	readdir(path: string): Promise<string[]>;
 	unlink(path: string): Promise<void>;
@@ -71,14 +73,6 @@ export function fileSystem(): FileSystem {
 	);
 }
 
-export async function pathExists(path: string): Promise<boolean> {
-	try {
-		return await fileSystem().exists(path);
-	} catch {
-		return false;
-	}
-}
-
 /** Remove a path, treating "already gone" as success (FR-062). */
 export async function removeIfPresent(path: string): Promise<boolean> {
 	try {

package/src/platform/location.ts

@@ -18,7 +18,6 @@ export const CATALOG_FILE = 'catalog.db';
 export const SCOPES_DIR = 'scopes';
 export const DATA_FILE = 'data.db';
 export const ARTIFACTS_DIR = 'artifacts';
-export const MARKER_FILE = '.localdb-marker';
 
 /** Normalize separators and collapse redundant segments. No `..` survives. */
 export function normalizePath(path: string): string {
@@ -46,7 +45,6 @@ export type StorageLayout = {
 	scopeDataPath(scopeKey: string): string;
 	scopeArtifactsPath(scopeKey: string): string;
 	artifactPath(scopeKey: string, attachmentId: string): string;
-	markerPath: string;
 };
 
 const SCOPE_KEY = /^[0-9a-f]{64}$/;
@@ -93,7 +91,6 @@ export function createLayout(documentDirectory: string, databaseName: string): S
 	return {
 		root,
 		catalogPath: `${root}/${CATALOG_FILE}`,
-		markerPath: `${root}/${MARKER_FILE}`,
 		scopePath,
 		scopeDataPath: scopeKey => `${scopePath(scopeKey)}/${DATA_FILE}`,
 		scopeArtifactsPath: scopeKey => `${scopePath(scopeKey)}/${ARTIFACTS_DIR}`,

package/src/query/budgets.ts

@@ -10,16 +10,42 @@ import { Budgets } from '../contract/config';
 import { DiagnosticCodes } from '../contract/errors';
 import type { StorableRecordData } from '../contract/record';
 import { raise } from '../diagnostics/errors';
-import { utf8Bytes } from '../platform/bytes';
+
+/**
+ * UTF-8 length of a string, counted rather than encoded: measuring a 256 KB
+ * record used to allocate a 256 KB byte array just to read its length.
+ */
+export function utf8Length(value: string): number {
+	let bytes = 0;
+	for (let index = 0; index < value.length; index += 1) {
+		const unit = value.charCodeAt(index);
+		if (unit < 0x80) {
+			bytes += 1;
+		} else if (unit < 0x800) {
+			bytes += 2;
+		} else if (unit >= 0xd800 && unit <= 0xdbff && index + 1 < value.length) {
+			// A surrogate pair is one four-byte code point.
+			bytes += 4;
+			index += 1;
+		} else {
+			bytes += 3;
+		}
+	}
+	return bytes;
+}
 
 /** UTF-8 byte length of the serialized payload — what actually hits disk. */
 export function serializedSize(data: StorableRecordData): number {
-	const json = JSON.stringify(data ?? {});
-	return utf8Bytes(json).length;
+	return utf8Length(JSON.stringify(data ?? {}));
 }
 
-export function assertRecordWithinBudget(collection: string, data: StorableRecordData): void {
-	const size = serializedSize(data);
+/**
+ * Checks the size budget and returns the serialized payload, so the write that
+ * follows stores this string instead of serializing the record a second time.
+ */
+export function assertRecordWithinBudget(collection: string, data: StorableRecordData): string {
+	const payload = JSON.stringify(data ?? {});
+	const size = utf8Length(payload);
 	if (size > Budgets.MAX_RECORD_BYTES) {
 		// Truncating would be silent data loss; failing opaquely would leave
 		// the consumer guessing. The size and the limit are both safe to
@@ -29,6 +55,7 @@ export function assertRecordWithinBudget(collection: string, data: StorableRecor
 			detail: { collection, sizeBytes: size, limitBytes: Budgets.MAX_RECORD_BYTES },
 		});
 	}
+	return payload;
 }
 
 export function assertBatchWithinBudget(count: number): void {