dbgate-sqlite-dumper 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +674 -0
- package/README.md +230 -0
- package/dist/better-sqlite3.cjs +178 -0
- package/dist/better-sqlite3.cjs.map +1 -0
- package/dist/better-sqlite3.d.cts +55 -0
- package/dist/better-sqlite3.d.ts +55 -0
- package/dist/better-sqlite3.js +140 -0
- package/dist/better-sqlite3.js.map +1 -0
- package/dist/index.cjs +3902 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +1893 -0
- package/dist/index.d.ts +1893 -0
- package/dist/index.js +3775 -0
- package/dist/index.js.map +1 -0
- package/dist/types-CTyJTzB6.d.cts +151 -0
- package/dist/types-CTyJTzB6.d.ts +151 -0
- package/docs/architecture.md +157 -0
- package/docs/better-sqlite3-adapter.md +83 -0
- package/docs/dump-api.md +253 -0
- package/docs/known-limitations.md +88 -0
- package/docs/native-compatibility.md +124 -0
- package/docs/restore-api.md +234 -0
- package/docs/round-trip-testing.md +84 -0
- package/docs/supported-data-types.md +74 -0
- package/docs/supported-objects.md +85 -0
- package/package.json +89 -0
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Client-agnostic SQLite connection abstraction.
|
|
3
|
+
*
|
|
4
|
+
* The core package never imports a Node.js driver directly. Callers provide
|
|
5
|
+
* a {@link SqliteConnection} (or a {@link SqliteConnectionSource} that can
|
|
6
|
+
* acquire one) implemented by an adapter such as
|
|
7
|
+
* `dbgate-sqlite-dumper/better-sqlite3`.
|
|
8
|
+
*
|
|
9
|
+
* SQLite is an embedded engine, so a "connection" here is one open database
|
|
10
|
+
* handle. The contract is deliberately the same shape the sibling dumper
|
|
11
|
+
* packages use for their network drivers, so an application can treat every
|
|
12
|
+
* dumper the same way.
|
|
13
|
+
*/
|
|
14
|
+
/** Scalar values accepted as bound query parameters (`?` placeholders). */
|
|
15
|
+
type SqliteParameterValue = string | number | bigint | Buffer | Uint8Array | null;
|
|
16
|
+
/** A single SQL statement plus its positional (`?`) parameters. */
|
|
17
|
+
interface SqliteQuery {
|
|
18
|
+
readonly sql: string;
|
|
19
|
+
/** Bound in order for each `?` placeholder. Adapters must never string-interpolate these. */
|
|
20
|
+
readonly parameters?: readonly SqliteParameterValue[];
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Scalar values that can appear in a returned row: SQLite's five storage
|
|
24
|
+
* classes as JavaScript sees them.
|
|
25
|
+
*
|
|
26
|
+
* `INTEGER` may arrive as `number` or `bigint` depending on the adapter; the
|
|
27
|
+
* core never relies on either for exactness. Row data is read through SQL
|
|
28
|
+
* that has SQLite itself render every numeric value as text (see
|
|
29
|
+
* `data/valueQuery.ts`), so a 64-bit integer or a `REAL`'s exact digits
|
|
30
|
+
* never pass through a JavaScript number on the way into the dump.
|
|
31
|
+
*/
|
|
32
|
+
type SqliteColumnValue = string | number | bigint | Buffer | Uint8Array | null;
|
|
33
|
+
/** A single result row, keyed by column name. */
|
|
34
|
+
interface SqliteRow {
|
|
35
|
+
readonly [column: string]: SqliteColumnValue;
|
|
36
|
+
}
|
|
37
|
+
/** Buffered result of a non-streaming query. */
|
|
38
|
+
interface SqliteQueryResult<Row extends SqliteRow = SqliteRow> {
|
|
39
|
+
readonly rows: readonly Row[];
|
|
40
|
+
/** Result column names, in order, when the adapter can report them. */
|
|
41
|
+
readonly columns?: readonly string[];
|
|
42
|
+
}
|
|
43
|
+
interface SqliteStreamOptions {
|
|
44
|
+
readonly signal?: AbortSignal;
|
|
45
|
+
/**
|
|
46
|
+
* Hint for adapters that fetch rows in batches. SQLite steps one row at a
|
|
47
|
+
* time in-process, so most adapters simply ignore it.
|
|
48
|
+
*/
|
|
49
|
+
readonly batchSize?: number;
|
|
50
|
+
}
|
|
51
|
+
/** Result of executing one statement through {@link SqliteConnection.execute}. */
|
|
52
|
+
interface SqliteExecResult {
|
|
53
|
+
/**
|
|
54
|
+
* Rows inserted, updated or deleted by *this* statement — `0` for DDL and
|
|
55
|
+
* for statements that return rows. Adapters must not report SQLite's
|
|
56
|
+
* `sqlite3_changes()` for a statement that is not DML, because it still
|
|
57
|
+
* holds the count from the previous DML statement.
|
|
58
|
+
*/
|
|
59
|
+
readonly changes: number;
|
|
60
|
+
}
|
|
61
|
+
/** Structured information about a SQLite error, extracted by an adapter. */
|
|
62
|
+
interface SqliteErrorInfo {
|
|
63
|
+
/** Symbolic (extended) result code, e.g. `SQLITE_CONSTRAINT_UNIQUE`. */
|
|
64
|
+
readonly code?: string;
|
|
65
|
+
/** Numeric (extended) result code, e.g. `2067`, when the adapter can report it. */
|
|
66
|
+
readonly errno?: number;
|
|
67
|
+
readonly message: string;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* One open SQLite database handle.
|
|
71
|
+
*
|
|
72
|
+
* Implementations must serialize statements sent through the same handle.
|
|
73
|
+
* In particular, while a {@link stream} is being consumed no other statement
|
|
74
|
+
* is sent on it — this package never interleaves them, because most
|
|
75
|
+
* synchronous drivers refuse to run a second statement while one is still
|
|
76
|
+
* stepping.
|
|
77
|
+
*/
|
|
78
|
+
interface SqliteConnection {
|
|
79
|
+
query<Row extends SqliteRow = SqliteRow>(query: SqliteQuery, signal?: AbortSignal): Promise<SqliteQueryResult<Row>>;
|
|
80
|
+
/** Streams rows without buffering the full result set in memory. */
|
|
81
|
+
stream<Row extends SqliteRow = SqliteRow>(query: SqliteQuery, options?: SqliteStreamOptions): AsyncIterable<Row>;
|
|
82
|
+
/**
|
|
83
|
+
* Executes one already-complete statement's SQL text with no parameter
|
|
84
|
+
* binding and no client-side rewriting.
|
|
85
|
+
*
|
|
86
|
+
* Restore routes every statement through this rather than `query()`
|
|
87
|
+
* because a dump's statement text must reach SQLite byte for byte: a driver
|
|
88
|
+
* that treats `?` or `:name` as a placeholder would corrupt any `INSERT`
|
|
89
|
+
* carrying one inside a string literal. Adapters that cannot make this
|
|
90
|
+
* guarantee may omit it; callers then fall back to `query()` with no
|
|
91
|
+
* parameters.
|
|
92
|
+
*/
|
|
93
|
+
execute?(sql: string, signal?: AbortSignal): Promise<SqliteExecResult>;
|
|
94
|
+
/** Extracts structured error fields from a driver error, for diagnostics. */
|
|
95
|
+
describeError?(error: unknown): SqliteErrorInfo | undefined;
|
|
96
|
+
/**
|
|
97
|
+
* Whether a transaction is open on this handle (`!sqlite3_get_autocommit`).
|
|
98
|
+
*
|
|
99
|
+
* SQLite has no SQL-level way to ask this, and it matters: a restore that
|
|
100
|
+
* stops part-way through a dump's own `BEGIN TRANSACTION` must roll it back
|
|
101
|
+
* before handing the handle back, or the caller inherits an open
|
|
102
|
+
* transaction holding a write lock on the file. Adapters that cannot answer
|
|
103
|
+
* may omit it; restore then tracks the script's own transaction statements
|
|
104
|
+
* instead.
|
|
105
|
+
*/
|
|
106
|
+
isInTransaction?(): boolean;
|
|
107
|
+
/**
|
|
108
|
+
* Turns `SQLITE_DBCONFIG_DEFENSIVE` on or off.
|
|
109
|
+
*
|
|
110
|
+
* A dump of a database containing virtual tables recreates them by
|
|
111
|
+
* inserting into `sqlite_schema` under `PRAGMA writable_schema=ON` — exactly
|
|
112
|
+
* as the native `.dump` does — and defensive mode forbids that. Restore
|
|
113
|
+
* uses this, when the adapter provides it, to lift the restriction for the
|
|
114
|
+
* duration of such a restore and put it back afterwards.
|
|
115
|
+
*/
|
|
116
|
+
setDefensive?(enabled: boolean): Promise<void>;
|
|
117
|
+
/**
|
|
118
|
+
* Requests cancellation of the currently executing statement, if any.
|
|
119
|
+
* Synchronous drivers cannot interrupt a statement mid-step; they
|
|
120
|
+
* implement this as a no-op, and cancellation then takes effect between
|
|
121
|
+
* rows and between statements.
|
|
122
|
+
*/
|
|
123
|
+
cancel(): Promise<void>;
|
|
124
|
+
}
|
|
125
|
+
/** A connection acquired from a pool-like source, plus its release callback. */
|
|
126
|
+
interface AcquiredSqliteConnection {
|
|
127
|
+
readonly connection: SqliteConnection;
|
|
128
|
+
/**
|
|
129
|
+
* Whether this handle is exclusively held for the duration of the
|
|
130
|
+
* operation. `false` for a bare {@link SqliteConnection} the caller handed
|
|
131
|
+
* over directly — it may be shared, so any state this package changes on
|
|
132
|
+
* it must be restored rather than assumed discarded.
|
|
133
|
+
*/
|
|
134
|
+
readonly dedicated: boolean;
|
|
135
|
+
/** Idempotent; safe to call more than once. */
|
|
136
|
+
release(): Promise<void>;
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Represents a resource that must be acquired to obtain one database
|
|
140
|
+
* handle, such as a pool of handles onto the same file. Direct
|
|
141
|
+
* {@link SqliteConnection} instances are borrowed by the library and are
|
|
142
|
+
* never closed by it.
|
|
143
|
+
*/
|
|
144
|
+
interface SqliteConnectionSource {
|
|
145
|
+
acquire(signal?: AbortSignal): Promise<AcquiredSqliteConnection>;
|
|
146
|
+
}
|
|
147
|
+
/** Anything the public API accepts in place of a database handle. */
|
|
148
|
+
type SqliteConnectionInput = SqliteConnection | SqliteConnectionSource;
|
|
149
|
+
declare function isSqliteConnectionSource(input: SqliteConnectionInput): input is SqliteConnectionSource;
|
|
150
|
+
|
|
151
|
+
export { type AcquiredSqliteConnection as A, type SqliteConnection as S, type SqliteConnectionInput as a, type SqliteExecResult as b, type SqliteRow as c, type SqliteErrorInfo as d, type SqliteColumnValue as e, type SqliteConnectionSource as f, type SqliteParameterValue as g, type SqliteQuery as h, type SqliteQueryResult as i, type SqliteStreamOptions as j, isSqliteConnectionSource as k };
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Client-agnostic SQLite connection abstraction.
|
|
3
|
+
*
|
|
4
|
+
* The core package never imports a Node.js driver directly. Callers provide
|
|
5
|
+
* a {@link SqliteConnection} (or a {@link SqliteConnectionSource} that can
|
|
6
|
+
* acquire one) implemented by an adapter such as
|
|
7
|
+
* `dbgate-sqlite-dumper/better-sqlite3`.
|
|
8
|
+
*
|
|
9
|
+
* SQLite is an embedded engine, so a "connection" here is one open database
|
|
10
|
+
* handle. The contract is deliberately the same shape the sibling dumper
|
|
11
|
+
* packages use for their network drivers, so an application can treat every
|
|
12
|
+
* dumper the same way.
|
|
13
|
+
*/
|
|
14
|
+
/** Scalar values accepted as bound query parameters (`?` placeholders). */
|
|
15
|
+
type SqliteParameterValue = string | number | bigint | Buffer | Uint8Array | null;
|
|
16
|
+
/** A single SQL statement plus its positional (`?`) parameters. */
|
|
17
|
+
interface SqliteQuery {
|
|
18
|
+
readonly sql: string;
|
|
19
|
+
/** Bound in order for each `?` placeholder. Adapters must never string-interpolate these. */
|
|
20
|
+
readonly parameters?: readonly SqliteParameterValue[];
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Scalar values that can appear in a returned row: SQLite's five storage
|
|
24
|
+
* classes as JavaScript sees them.
|
|
25
|
+
*
|
|
26
|
+
* `INTEGER` may arrive as `number` or `bigint` depending on the adapter; the
|
|
27
|
+
* core never relies on either for exactness. Row data is read through SQL
|
|
28
|
+
* that has SQLite itself render every numeric value as text (see
|
|
29
|
+
* `data/valueQuery.ts`), so a 64-bit integer or a `REAL`'s exact digits
|
|
30
|
+
* never pass through a JavaScript number on the way into the dump.
|
|
31
|
+
*/
|
|
32
|
+
type SqliteColumnValue = string | number | bigint | Buffer | Uint8Array | null;
|
|
33
|
+
/** A single result row, keyed by column name. */
|
|
34
|
+
interface SqliteRow {
|
|
35
|
+
readonly [column: string]: SqliteColumnValue;
|
|
36
|
+
}
|
|
37
|
+
/** Buffered result of a non-streaming query. */
|
|
38
|
+
interface SqliteQueryResult<Row extends SqliteRow = SqliteRow> {
|
|
39
|
+
readonly rows: readonly Row[];
|
|
40
|
+
/** Result column names, in order, when the adapter can report them. */
|
|
41
|
+
readonly columns?: readonly string[];
|
|
42
|
+
}
|
|
43
|
+
interface SqliteStreamOptions {
|
|
44
|
+
readonly signal?: AbortSignal;
|
|
45
|
+
/**
|
|
46
|
+
* Hint for adapters that fetch rows in batches. SQLite steps one row at a
|
|
47
|
+
* time in-process, so most adapters simply ignore it.
|
|
48
|
+
*/
|
|
49
|
+
readonly batchSize?: number;
|
|
50
|
+
}
|
|
51
|
+
/** Result of executing one statement through {@link SqliteConnection.execute}. */
|
|
52
|
+
interface SqliteExecResult {
|
|
53
|
+
/**
|
|
54
|
+
* Rows inserted, updated or deleted by *this* statement — `0` for DDL and
|
|
55
|
+
* for statements that return rows. Adapters must not report SQLite's
|
|
56
|
+
* `sqlite3_changes()` for a statement that is not DML, because it still
|
|
57
|
+
* holds the count from the previous DML statement.
|
|
58
|
+
*/
|
|
59
|
+
readonly changes: number;
|
|
60
|
+
}
|
|
61
|
+
/** Structured information about a SQLite error, extracted by an adapter. */
|
|
62
|
+
interface SqliteErrorInfo {
|
|
63
|
+
/** Symbolic (extended) result code, e.g. `SQLITE_CONSTRAINT_UNIQUE`. */
|
|
64
|
+
readonly code?: string;
|
|
65
|
+
/** Numeric (extended) result code, e.g. `2067`, when the adapter can report it. */
|
|
66
|
+
readonly errno?: number;
|
|
67
|
+
readonly message: string;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* One open SQLite database handle.
|
|
71
|
+
*
|
|
72
|
+
* Implementations must serialize statements sent through the same handle.
|
|
73
|
+
* In particular, while a {@link stream} is being consumed no other statement
|
|
74
|
+
* is sent on it — this package never interleaves them, because most
|
|
75
|
+
* synchronous drivers refuse to run a second statement while one is still
|
|
76
|
+
* stepping.
|
|
77
|
+
*/
|
|
78
|
+
interface SqliteConnection {
|
|
79
|
+
query<Row extends SqliteRow = SqliteRow>(query: SqliteQuery, signal?: AbortSignal): Promise<SqliteQueryResult<Row>>;
|
|
80
|
+
/** Streams rows without buffering the full result set in memory. */
|
|
81
|
+
stream<Row extends SqliteRow = SqliteRow>(query: SqliteQuery, options?: SqliteStreamOptions): AsyncIterable<Row>;
|
|
82
|
+
/**
|
|
83
|
+
* Executes one already-complete statement's SQL text with no parameter
|
|
84
|
+
* binding and no client-side rewriting.
|
|
85
|
+
*
|
|
86
|
+
* Restore routes every statement through this rather than `query()`
|
|
87
|
+
* because a dump's statement text must reach SQLite byte for byte: a driver
|
|
88
|
+
* that treats `?` or `:name` as a placeholder would corrupt any `INSERT`
|
|
89
|
+
* carrying one inside a string literal. Adapters that cannot make this
|
|
90
|
+
* guarantee may omit it; callers then fall back to `query()` with no
|
|
91
|
+
* parameters.
|
|
92
|
+
*/
|
|
93
|
+
execute?(sql: string, signal?: AbortSignal): Promise<SqliteExecResult>;
|
|
94
|
+
/** Extracts structured error fields from a driver error, for diagnostics. */
|
|
95
|
+
describeError?(error: unknown): SqliteErrorInfo | undefined;
|
|
96
|
+
/**
|
|
97
|
+
* Whether a transaction is open on this handle (`!sqlite3_get_autocommit`).
|
|
98
|
+
*
|
|
99
|
+
* SQLite has no SQL-level way to ask this, and it matters: a restore that
|
|
100
|
+
* stops part-way through a dump's own `BEGIN TRANSACTION` must roll it back
|
|
101
|
+
* before handing the handle back, or the caller inherits an open
|
|
102
|
+
* transaction holding a write lock on the file. Adapters that cannot answer
|
|
103
|
+
* may omit it; restore then tracks the script's own transaction statements
|
|
104
|
+
* instead.
|
|
105
|
+
*/
|
|
106
|
+
isInTransaction?(): boolean;
|
|
107
|
+
/**
|
|
108
|
+
* Turns `SQLITE_DBCONFIG_DEFENSIVE` on or off.
|
|
109
|
+
*
|
|
110
|
+
* A dump of a database containing virtual tables recreates them by
|
|
111
|
+
* inserting into `sqlite_schema` under `PRAGMA writable_schema=ON` — exactly
|
|
112
|
+
* as the native `.dump` does — and defensive mode forbids that. Restore
|
|
113
|
+
* uses this, when the adapter provides it, to lift the restriction for the
|
|
114
|
+
* duration of such a restore and put it back afterwards.
|
|
115
|
+
*/
|
|
116
|
+
setDefensive?(enabled: boolean): Promise<void>;
|
|
117
|
+
/**
|
|
118
|
+
* Requests cancellation of the currently executing statement, if any.
|
|
119
|
+
* Synchronous drivers cannot interrupt a statement mid-step; they
|
|
120
|
+
* implement this as a no-op, and cancellation then takes effect between
|
|
121
|
+
* rows and between statements.
|
|
122
|
+
*/
|
|
123
|
+
cancel(): Promise<void>;
|
|
124
|
+
}
|
|
125
|
+
/** A connection acquired from a pool-like source, plus its release callback. */
|
|
126
|
+
interface AcquiredSqliteConnection {
|
|
127
|
+
readonly connection: SqliteConnection;
|
|
128
|
+
/**
|
|
129
|
+
* Whether this handle is exclusively held for the duration of the
|
|
130
|
+
* operation. `false` for a bare {@link SqliteConnection} the caller handed
|
|
131
|
+
* over directly — it may be shared, so any state this package changes on
|
|
132
|
+
* it must be restored rather than assumed discarded.
|
|
133
|
+
*/
|
|
134
|
+
readonly dedicated: boolean;
|
|
135
|
+
/** Idempotent; safe to call more than once. */
|
|
136
|
+
release(): Promise<void>;
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Represents a resource that must be acquired to obtain one database
|
|
140
|
+
* handle, such as a pool of handles onto the same file. Direct
|
|
141
|
+
* {@link SqliteConnection} instances are borrowed by the library and are
|
|
142
|
+
* never closed by it.
|
|
143
|
+
*/
|
|
144
|
+
interface SqliteConnectionSource {
|
|
145
|
+
acquire(signal?: AbortSignal): Promise<AcquiredSqliteConnection>;
|
|
146
|
+
}
|
|
147
|
+
/** Anything the public API accepts in place of a database handle. */
|
|
148
|
+
type SqliteConnectionInput = SqliteConnection | SqliteConnectionSource;
|
|
149
|
+
declare function isSqliteConnectionSource(input: SqliteConnectionInput): input is SqliteConnectionSource;
|
|
150
|
+
|
|
151
|
+
export { type AcquiredSqliteConnection as A, type SqliteConnection as S, type SqliteConnectionInput as a, type SqliteExecResult as b, type SqliteRow as c, type SqliteErrorInfo as d, type SqliteColumnValue as e, type SqliteConnectionSource as f, type SqliteParameterValue as g, type SqliteQuery as h, type SqliteQueryResult as i, type SqliteStreamOptions as j, isSqliteConnectionSource as k };
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
# Architecture
|
|
2
|
+
|
|
3
|
+
The package is a stack of layers, each usable on its own. Nothing above the `connection`
|
|
4
|
+
layer knows what driver is in use; nothing below the `api` layer knows about a dump as a
|
|
5
|
+
whole.
|
|
6
|
+
|
|
7
|
+
```
|
|
8
|
+
┌──────────────────────────────────────────┐
|
|
9
|
+
api/ │ dumpSqlite() · restoreSqlDump() │ orchestration
|
|
10
|
+
└──────────────────────────────────────────┘
|
|
11
|
+
│ │ │
|
|
12
|
+
introspection/ ──────────┘ │ └────── restore/
|
|
13
|
+
sqlite_schema + pragmas → model │ sqlite3_complete + shell
|
|
14
|
+
│ line rules → statements
|
|
15
|
+
archive/ ─────── plan: what & in what order
|
|
16
|
+
renderer/ ────── model + plan → plain SQL text preflight/ compatibility/
|
|
17
|
+
data/ ────────── rows → INSERT statements selection/ security/
|
|
18
|
+
writer/ ──────── text/bytes → Writable version/ model/
|
|
19
|
+
│
|
|
20
|
+
┌──────────────────────────────────────────┐
|
|
21
|
+
connection/ │ SqliteConnection (driver-agnostic) │
|
|
22
|
+
└──────────────────────────────────────────┘
|
|
23
|
+
│
|
|
24
|
+
better-sqlite3.ts ─ the only module that knows better-sqlite3 exists
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
This mirrors `dbgate-mysql-dumper`, `dbgate-pg-dumper`, `dbgate-mssql-dumper` and
|
|
28
|
+
`dbgate-redis-dumper`; the SQLite-specific differences are called out below.
|
|
29
|
+
|
|
30
|
+
## `connection/` — the driver boundary
|
|
31
|
+
|
|
32
|
+
`SqliteConnection` is the whole contract between this package and a driver: `query()`,
|
|
33
|
+
`stream()`, and optional `execute()`, `describeError()`, `isInTransaction()` and
|
|
34
|
+
`setDefensive()`, plus `cancel()`. The core never imports a driver, and
|
|
35
|
+
`tests/packageBoundaries.test.ts` fails if it starts to.
|
|
36
|
+
|
|
37
|
+
SQLite is embedded, so a "connection" is one open database handle. The contract keeps the
|
|
38
|
+
shape of the network drivers' in the sibling packages — including `SqliteConnectionSource`
|
|
39
|
+
for pool-like inputs — so an application can drive every dumper the same way.
|
|
40
|
+
|
|
41
|
+
### Why one read transaction
|
|
42
|
+
|
|
43
|
+
A dump runs entirely inside one transaction on one handle (`session.ts`), opened as a
|
|
44
|
+
`SAVEPOINT` so that it nests inside a transaction the caller already has, and with one read
|
|
45
|
+
issued immediately so the snapshot is taken at the start rather than by whichever catalog
|
|
46
|
+
query happens to run first. In WAL mode that is a true snapshot; in rollback-journal mode it
|
|
47
|
+
holds a `SHARED` lock. The native shell does the same (`SAVEPOINT dump`). It is released
|
|
48
|
+
without the caller's `AbortSignal`, so a cancelled dump still ends it.
|
|
49
|
+
|
|
50
|
+
## `introspection/` — catalog to model
|
|
51
|
+
|
|
52
|
+
Everything comes from `sqlite_schema` (read as `sqlite_master`, which every release
|
|
53
|
+
accepts), in rowid order, plus the table-valued pragma functions — `table_xinfo`,
|
|
54
|
+
`index_list`, `index_xinfo`, `foreign_key_list`, `table_list` — with a fallback to plain
|
|
55
|
+
`PRAGMA` statements on libraries too old for them. Tables are classified as ordinary,
|
|
56
|
+
virtual, shadow (a virtual table's storage) or system (`sqlite_*`).
|
|
57
|
+
|
|
58
|
+
### Why the DDL is not reconstructed
|
|
59
|
+
|
|
60
|
+
SQLite keeps every `CREATE` statement verbatim, and the native `.dump` writes that text.
|
|
61
|
+
Reconstructing DDL from the column model would lose comments, spelling, formatting and any
|
|
62
|
+
clause the model does not know about, for no gain. The model exists for everything else —
|
|
63
|
+
planning, selection, `INSERT` column lists, compatibility checks, diagnostics — which is why
|
|
64
|
+
`renderPlainSql` is a pure function of it.
|
|
65
|
+
|
|
66
|
+
## `archive/` — the plan
|
|
67
|
+
|
|
68
|
+
`inspectDumpArchive` turns a model into ordered `ArchiveEntry` objects. It is pure: no SQL
|
|
69
|
+
text, no streams, no connection.
|
|
70
|
+
|
|
71
|
+
The order is the native `.dump`'s — creation order — not a topological sort. Creation order
|
|
72
|
+
is valid for everything SQLite checks at `CREATE` time (an index after its table, a trigger
|
|
73
|
+
after its table or view), SQLite resolves view and trigger bodies lazily, and the dump's
|
|
74
|
+
`PRAGMA foreign_keys=OFF` makes table order irrelevant to foreign keys. Dependencies are
|
|
75
|
+
still recorded and then _verified_ against that order:
|
|
76
|
+
|
|
77
|
+
- **`hard`** — a trigger depends on its table's data this way: created before the rows, it
|
|
78
|
+
would fire for each one.
|
|
79
|
+
- **`preference`** — every foreign key. Recording them as hard edges would report a false
|
|
80
|
+
cycle for exactly the circular schemas that restore perfectly well.
|
|
81
|
+
|
|
82
|
+
## `renderer/` — model to text
|
|
83
|
+
|
|
84
|
+
A port of the parts of `shell.c` that shape `.dump` output — `printSchemaLine()`,
|
|
85
|
+
`run_table_dump_query()`, the virtual-table `INSERT INTO sqlite_schema` — driven by the
|
|
86
|
+
archive, with row data arriving through an `onTableData` hook so the renderer never needs a
|
|
87
|
+
connection. `sqlite3_complete()` itself is ported (in `restore/complete.ts`) because
|
|
88
|
+
`printSchemaLine()` calls it to decide how to terminate DDL that ends in a comment.
|
|
89
|
+
|
|
90
|
+
## `data/` — rows to `INSERT`
|
|
91
|
+
|
|
92
|
+
`exportTableDataAsInserts` streams one table in constant memory.
|
|
93
|
+
|
|
94
|
+
### Why SQLite renders the values
|
|
95
|
+
|
|
96
|
+
The `SELECT` it runs does not fetch column values; it fetches each value's storage class and
|
|
97
|
+
a representation computed _by SQLite_: integers as text, reals as the finished literal —
|
|
98
|
+
using SQLite's own `printf('%!.20g')`, the formatter the shell uses — and text as its stored
|
|
99
|
+
bytes. Three problems disappear at once: 64-bit integers never become JavaScript numbers;
|
|
100
|
+
`REAL` digits match the shell's exactly, which no JavaScript formatting could; and text that
|
|
101
|
+
is not valid UTF-8 is not "repaired" by the driver's decoder. The core is then exact with
|
|
102
|
+
any driver that can return strings and bytes.
|
|
103
|
+
|
|
104
|
+
`SqlChunkBuilder` keeps `(string | Buffer)[]` parts and only falls back to `Buffer.concat`
|
|
105
|
+
once it has been handed bytes, so the all-text case stays on the string fast path.
|
|
106
|
+
|
|
107
|
+
## `restore/` — text to statements
|
|
108
|
+
|
|
109
|
+
### The parser
|
|
110
|
+
|
|
111
|
+
`SqlStatementParser` is an incremental scanner over **bytes**: a dump is not necessarily
|
|
112
|
+
valid UTF-8, and `latin1` is a bijection between bytes and code units, so nothing is lost
|
|
113
|
+
and every character the scanner reacts to is ASCII.
|
|
114
|
+
|
|
115
|
+
It has two layers, both ported: `sqlite3_complete()`'s eight-state machine, which decides
|
|
116
|
+
when a `;` completes a statement (and keeps trigger bodies whole), and the shell's
|
|
117
|
+
`process_input()` line rules — dot-commands and `#` comments at column 0 when nothing is
|
|
118
|
+
pending, `GO`/`/` terminator lines checked with the shell's own `line_is_complete()`
|
|
119
|
+
condition, and the `\r` its line reader drops before each `\n`.
|
|
120
|
+
|
|
121
|
+
Anything whose meaning depends on the next chunk — a `-` or `/` that may open a comment, a
|
|
122
|
+
`*` that may close one, a `\r` that may precede `\n`, a line start that may be a `GO` line,
|
|
123
|
+
a word that may be a keyword — is carried or kept in state, never guessed.
|
|
124
|
+
`tests/statementParser.test.ts` checks identical output at every chunk size and every split
|
|
125
|
+
point.
|
|
126
|
+
|
|
127
|
+
### Session cleanup
|
|
128
|
+
|
|
129
|
+
`RestoreSessionState` follows only classified top-level statements (transaction control,
|
|
130
|
+
`PRAGMA foreign_keys`, `PRAGMA writable_schema`, writes to the schema table), never text
|
|
131
|
+
inside rows or trigger bodies, and puts back what the script changed: it rolls back a
|
|
132
|
+
transaction the script opened and left open, resets `writable_schema`, and returns
|
|
133
|
+
`foreign_keys` to its previous value — which a native dump never does, because it is written
|
|
134
|
+
for a shell that exits.
|
|
135
|
+
|
|
136
|
+
## `writer/` — text and bytes out
|
|
137
|
+
|
|
138
|
+
`DumpWriter.write` accepts `string | Buffer` for the reason above. `StreamDumpWriter` honours
|
|
139
|
+
backpressure by gating on `write()`'s return value with the `drain` listener attached in the
|
|
140
|
+
same tick, and never calls `end()` on a caller-owned stream.
|
|
141
|
+
|
|
142
|
+
## `security/` — quoting and escaping
|
|
143
|
+
|
|
144
|
+
Two quoting functions, for two jobs. `quoteIdentifier` always double-quotes, and is used for
|
|
145
|
+
every query this package runs. `quoteIdentifierIfNeeded` reproduces the shell's
|
|
146
|
+
`quoteChar()` — quote only non-identifiers and SQLite's 147 keywords — and is used only
|
|
147
|
+
where output must match the native layout. Literal rendering ports
|
|
148
|
+
`output_quoted_escaped_string()`, including `unused_string()`'s placeholder choice.
|
|
149
|
+
|
|
150
|
+
## Testing strategy
|
|
151
|
+
|
|
152
|
+
| Suite | Needs | What it proves |
|
|
153
|
+
| -------------- | --------------- | ------------------------------------------------------------------- |
|
|
154
|
+
| `tests/` | nothing | Every layer, against in-memory `better-sqlite3` databases. |
|
|
155
|
+
| `integration/` | `sqlite3` shell | Byte identity, the four-way matrix, shell-script parity, streaming. |
|
|
156
|
+
|
|
157
|
+
See [round-trip-testing.md](round-trip-testing.md).
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# The `better-sqlite3` adapter
|
|
2
|
+
|
|
3
|
+
```ts
|
|
4
|
+
import { fromBetterSqlite3, connectBetterSqlite3 } from 'dbgate-sqlite-dumper/better-sqlite3';
|
|
5
|
+
```
|
|
6
|
+
|
|
7
|
+
The only module that knows `better-sqlite3` exists, reachable only through this separate
|
|
8
|
+
entry point. `better-sqlite3` is an optional peer dependency: the type import costs nothing
|
|
9
|
+
at runtime and the value import is dynamic, so the core package — and this module — load
|
|
10
|
+
without it installed. `better-sqlite3` is the driver DbGate's own SQLite plugin uses.
|
|
11
|
+
|
|
12
|
+
## Ownership
|
|
13
|
+
|
|
14
|
+
| Function | Who closes the database |
|
|
15
|
+
| --------------------------------------- | ----------------------------------------- |
|
|
16
|
+
| `fromBetterSqlite3(database)` | The caller. The database is **borrowed**. |
|
|
17
|
+
| `connectBetterSqlite3(filename, opts?)` | The returned `close()`. Idempotent. |
|
|
18
|
+
|
|
19
|
+
`connectBetterSqlite3` passes `opts` straight to the `Database` constructor, so
|
|
20
|
+
`{ readonly: true }` for a dump source, `{ fileMustExist: true }` and the rest all work.
|
|
21
|
+
|
|
22
|
+
## Behaviour
|
|
23
|
+
|
|
24
|
+
- **Integers are read as `bigint`** (`safeIntegers(true)`) for every catalog query, so no
|
|
25
|
+
value is ever rounded. Row data does not depend on this: it is read through SQL that has
|
|
26
|
+
SQLite render every value as text or bytes (see
|
|
27
|
+
[supported-data-types.md](supported-data-types.md)).
|
|
28
|
+
- **`stream()` steps the statement with `iterate()`**, one row at a time, yielding between
|
|
29
|
+
rows — constant memory for a table of any size. While a stream is open, the database
|
|
30
|
+
cannot run another statement; this package never asks it to.
|
|
31
|
+
- **`execute()` reports a statement's own row count** (`changes`), 0 for DDL — not the
|
|
32
|
+
count left over from the previous `INSERT`. Text holding several statements, which the
|
|
33
|
+
parser does not produce but other callers might, falls back to `exec()`.
|
|
34
|
+
- **`isInTransaction()`** is `database.inTransaction`, which is what lets a restore roll
|
|
35
|
+
back exactly the transaction the script opened.
|
|
36
|
+
- **`setDefensive()`** is `unsafeMode()`, which is how `better-sqlite3` exposes
|
|
37
|
+
`SQLITE_DBCONFIG_DEFENSIVE` — on by default in `better-sqlite3`. A restore lifts it only
|
|
38
|
+
for a dump that writes `sqlite_schema` (a virtual-table dump), and puts it back. Unsafe
|
|
39
|
+
mode also relaxes the driver's own guard against writing while iterating; this package
|
|
40
|
+
never does that.
|
|
41
|
+
- **`cancel()` is a no-op.** `better-sqlite3` runs each statement synchronously to
|
|
42
|
+
completion, so cancellation takes effect between rows and between statements — prompt,
|
|
43
|
+
since every statement a dump or a restore runs is short.
|
|
44
|
+
|
|
45
|
+
## Foreign keys
|
|
46
|
+
|
|
47
|
+
`better-sqlite3` turns `foreign_keys` **on** for every database it opens; the `sqlite3`
|
|
48
|
+
shell leaves it off. A full dump turns enforcement off itself, but a data-only dump does
|
|
49
|
+
not, so loading one whose tables reference each other needs `disableForeignKeys: true`
|
|
50
|
+
(see [restore-api.md](restore-api.md)).
|
|
51
|
+
|
|
52
|
+
## Statistics tables
|
|
53
|
+
|
|
54
|
+
`better-sqlite3` is built with `SQLITE_ENABLE_STAT4`, so `ANALYZE` — including the
|
|
55
|
+
`ANALYZE sqlite_schema;` a dump runs — creates a `sqlite_stat4` table. A dump of a database
|
|
56
|
+
that has one does not restore on a build without STAT4 (the native `.dump` of it fails the
|
|
57
|
+
same way); `preflightRestore` reports it as `statistics-table-unsupported`.
|
|
58
|
+
|
|
59
|
+
## Other drivers
|
|
60
|
+
|
|
61
|
+
Anything that can run SQL can be adapted by implementing `SqliteConnection`:
|
|
62
|
+
|
|
63
|
+
```ts
|
|
64
|
+
interface SqliteConnection {
|
|
65
|
+
query(
|
|
66
|
+
query: { sql: string; parameters?: readonly unknown[] },
|
|
67
|
+
signal?,
|
|
68
|
+
): Promise<{ rows: readonly Row[] }>;
|
|
69
|
+
stream(query, options?): AsyncIterable<Row>;
|
|
70
|
+
execute?(sql: string, signal?): Promise<{ changes: number }>; // run text verbatim, no parameter parsing
|
|
71
|
+
describeError?(error): { code?: string; errno?: number; message: string } | undefined;
|
|
72
|
+
isInTransaction?(): boolean;
|
|
73
|
+
setDefensive?(enabled: boolean): Promise<void>;
|
|
74
|
+
cancel(): Promise<void>;
|
|
75
|
+
}
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
`query` and `stream` return rows as objects keyed by column name. Values may be any of
|
|
79
|
+
SQLite's storage classes as the driver represents them; the core only relies on text,
|
|
80
|
+
bytes and `null`, which every driver delivers exactly. The optional members degrade
|
|
81
|
+
gracefully: without `execute`, statements go through `query`; without `isInTransaction`,
|
|
82
|
+
restore tracks the script's own `BEGIN`/`COMMIT`; without `setDefensive`, a virtual-table
|
|
83
|
+
dump restores only onto a handle that is not in defensive mode, and preflight says so.
|