@peerbit/indexer-sqlite3 3.0.0-e6ea5c0 → 3.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/sqlite3/sqlite3.worker.min.js +168 -52
- package/dist/index.min.js +277 -53
- package/dist/index.min.js.map +3 -3
- package/dist/src/engine.d.ts +9 -3
- package/dist/src/engine.d.ts.map +1 -1
- package/dist/src/engine.js +75 -9
- package/dist/src/engine.js.map +1 -1
- package/dist/src/index.d.ts +7 -3
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +5 -4
- package/dist/src/index.js.map +1 -1
- package/dist/src/schema.d.ts +5 -1
- package/dist/src/schema.d.ts.map +1 -1
- package/dist/src/schema.js +42 -12
- package/dist/src/schema.js.map +1 -1
- package/dist/src/sqlite3-messages.worker.d.ts +47 -8
- package/dist/src/sqlite3-messages.worker.d.ts.map +1 -1
- package/dist/src/sqlite3-messages.worker.js +42 -3
- package/dist/src/sqlite3-messages.worker.js.map +1 -1
- package/dist/src/sqlite3.browser.d.ts +22 -1
- package/dist/src/sqlite3.browser.d.ts.map +1 -1
- package/dist/src/sqlite3.browser.js +138 -31
- package/dist/src/sqlite3.browser.js.map +1 -1
- package/dist/src/sqlite3.d.ts +4 -1
- package/dist/src/sqlite3.d.ts.map +1 -1
- package/dist/src/sqlite3.js +14 -5
- package/dist/src/sqlite3.js.map +1 -1
- package/dist/src/sqlite3.wasm.d.ts +4 -1
- package/dist/src/sqlite3.wasm.d.ts.map +1 -1
- package/dist/src/sqlite3.wasm.js +15 -3
- package/dist/src/sqlite3.wasm.js.map +1 -1
- package/dist/src/sqlite3.worker.js +146 -53
- package/dist/src/sqlite3.worker.js.map +1 -1
- package/dist/src/utils.d.ts +1 -0
- package/dist/src/utils.d.ts.map +1 -1
- package/dist/src/utils.js +11 -0
- package/dist/src/utils.js.map +1 -1
- package/package.json +10 -9
- package/src/engine.ts +99 -11
- package/src/index.ts +39 -4
- package/src/schema.ts +67 -11
- package/src/sqlite3-messages.worker.ts +104 -10
- package/src/sqlite3.browser.ts +246 -88
- package/src/sqlite3.ts +19 -5
- package/src/sqlite3.wasm.ts +25 -3
- package/src/sqlite3.worker.ts +170 -49
- package/src/utils.ts +14 -0
package/src/schema.ts
CHANGED
|
@@ -817,7 +817,11 @@ const isUint8ArrayType = (type: FieldType) => {
|
|
|
817
817
|
};
|
|
818
818
|
|
|
819
819
|
export const insert = async (
|
|
820
|
-
insertFn: (
|
|
820
|
+
insertFn: (
|
|
821
|
+
values: any[],
|
|
822
|
+
table: Table,
|
|
823
|
+
options?: { requireId?: boolean },
|
|
824
|
+
) => Promise<any> | any,
|
|
821
825
|
obj: Record<string, any>,
|
|
822
826
|
tables: Map<string, Table>,
|
|
823
827
|
table: Table,
|
|
@@ -827,6 +831,12 @@ export const insert = async (
|
|
|
827
831
|
) => Promise<void> | void | number,
|
|
828
832
|
parentId: any = undefined,
|
|
829
833
|
index?: number,
|
|
834
|
+
options?: {
|
|
835
|
+
insertSimpleVecRows?: (
|
|
836
|
+
rows: any[][],
|
|
837
|
+
table: Table,
|
|
838
|
+
) => Promise<void> | void;
|
|
839
|
+
},
|
|
830
840
|
): Promise<void> => {
|
|
831
841
|
const bindableValues: any[] = [];
|
|
832
842
|
let nestedCallbacks: ((id: any) => Promise<void>)[] = [];
|
|
@@ -836,6 +846,14 @@ export const insert = async (
|
|
|
836
846
|
? handleNestedCallback
|
|
837
847
|
: (fn) => nestedCallbacks.push(fn);
|
|
838
848
|
|
|
849
|
+
const toInsertValue = (item: any, subTable: Table) =>
|
|
850
|
+
typeof item === "function" && item instanceof Uint8Array === false
|
|
851
|
+
? item
|
|
852
|
+
: subTable.isSimpleValue
|
|
853
|
+
? // eslint-disable-next-line new-cap
|
|
854
|
+
new subTable.ctor(item)
|
|
855
|
+
: Object.assign(Object.create(subTable.ctor.prototype), item);
|
|
856
|
+
|
|
839
857
|
const handleElement = async (
|
|
840
858
|
item: any,
|
|
841
859
|
field: Field,
|
|
@@ -846,18 +864,14 @@ export const insert = async (
|
|
|
846
864
|
|
|
847
865
|
await insert(
|
|
848
866
|
insertFn,
|
|
849
|
-
|
|
850
|
-
? item
|
|
851
|
-
: subTable.isSimpleValue
|
|
852
|
-
? // eslint-disable-next-line new-cap
|
|
853
|
-
new subTable.ctor(item)
|
|
854
|
-
: Object.assign(Object.create(subTable.ctor.prototype), item),
|
|
867
|
+
toInsertValue(item, subTable),
|
|
855
868
|
tables,
|
|
856
869
|
subTable,
|
|
857
870
|
getSchema(subTable.ctor).fields,
|
|
858
871
|
handleNestedCallback,
|
|
859
872
|
parentId,
|
|
860
873
|
index,
|
|
874
|
+
options,
|
|
861
875
|
);
|
|
862
876
|
};
|
|
863
877
|
|
|
@@ -868,6 +882,39 @@ export const insert = async (
|
|
|
868
882
|
) => {
|
|
869
883
|
if (Array.isArray(obj[field.key])) {
|
|
870
884
|
const arr = obj[field.key];
|
|
885
|
+
const firstItem = arr.find((item: any) => item != null);
|
|
886
|
+
if (
|
|
887
|
+
parentId != null &&
|
|
888
|
+
arr.length > 1 &&
|
|
889
|
+
firstItem != null &&
|
|
890
|
+
options?.insertSimpleVecRows
|
|
891
|
+
) {
|
|
892
|
+
const subTable = getTableFromValue(table, tables, field, firstItem);
|
|
893
|
+
if (subTable.isSimpleValue) {
|
|
894
|
+
const rows: any[][] = [];
|
|
895
|
+
for (let i = 0; i < arr.length; i++) {
|
|
896
|
+
const item = arr[i];
|
|
897
|
+
await insert(
|
|
898
|
+
(values) => {
|
|
899
|
+
rows.push(values);
|
|
900
|
+
return undefined;
|
|
901
|
+
},
|
|
902
|
+
toInsertValue(item, subTable),
|
|
903
|
+
tables,
|
|
904
|
+
subTable,
|
|
905
|
+
getSchema(subTable.ctor).fields,
|
|
906
|
+
handleNestedCallback,
|
|
907
|
+
parentId,
|
|
908
|
+
i,
|
|
909
|
+
options,
|
|
910
|
+
);
|
|
911
|
+
}
|
|
912
|
+
if (rows.length > 0) {
|
|
913
|
+
await options.insertSimpleVecRows(rows, subTable);
|
|
914
|
+
}
|
|
915
|
+
return;
|
|
916
|
+
}
|
|
917
|
+
}
|
|
871
918
|
for (let i = 0; i < arr.length; i++) {
|
|
872
919
|
const item = arr[i];
|
|
873
920
|
await handleElement(item, field, parentId, i);
|
|
@@ -945,6 +992,7 @@ export const insert = async (
|
|
|
945
992
|
(fn) => nestedCallbacks.push(fn),
|
|
946
993
|
undefined, // parentId is not defined here, we are inserting a nested object inline
|
|
947
994
|
undefined, // index is not defined here, we are inserting a nested object inline
|
|
995
|
+
options,
|
|
948
996
|
);
|
|
949
997
|
/* await insert(, obj[field.key], tables, subTable, getSchema(unwrappedType).fields, parentId, index); */
|
|
950
998
|
} else {
|
|
@@ -963,7 +1011,9 @@ export const insert = async (
|
|
|
963
1011
|
await handleNestedCallback!((id) => handleNested(nested, isOptional, id));
|
|
964
1012
|
}
|
|
965
1013
|
|
|
966
|
-
const thisId = await insertFn(bindableValues, table
|
|
1014
|
+
const thisId = await insertFn(bindableValues, table, {
|
|
1015
|
+
requireId: nestedCallbacks.length > 0,
|
|
1016
|
+
});
|
|
967
1017
|
if (table.primary === false && nestedCallbacks.length > 0) {
|
|
968
1018
|
throw new Error("Unexpected");
|
|
969
1019
|
}
|
|
@@ -1875,9 +1925,15 @@ const _buildJoin = (
|
|
|
1875
1925
|
|
|
1876
1926
|
if (table!.columns.length > 0) {
|
|
1877
1927
|
const usedColumns = removeDuplicatesOrdered(table!.columns);
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1928
|
+
const usesImplicitPrimaryKeyIndex =
|
|
1929
|
+
table.type === "root" &&
|
|
1930
|
+
table.table.primary !== false &&
|
|
1931
|
+
usedColumns.length === 1 &&
|
|
1932
|
+
usedColumns[0] === table.table.primary;
|
|
1933
|
+
indexedBy =
|
|
1934
|
+
options?.planner && !usesImplicitPrimaryKeyIndex
|
|
1935
|
+
? ` INDEXED BY ${options.planner.resolveIndex(table.table.name, usedColumns)} `
|
|
1936
|
+
: "";
|
|
1881
1937
|
}
|
|
1882
1938
|
|
|
1883
1939
|
if (table.type !== "root") {
|
|
@@ -1,14 +1,27 @@
|
|
|
1
1
|
import { fromBase64, toBase64 } from "@peerbit/crypto";
|
|
2
2
|
|
|
3
|
+
export type SqliteWorkerProtocol = "legacy" | "clone";
|
|
4
|
+
export type SQLiteSynchronousMode = "FULL" | "NORMAL" | "OFF";
|
|
5
|
+
export type SQLiteLockingMode = "NORMAL" | "EXCLUSIVE";
|
|
6
|
+
export type SQLiteTempStoreMode = "DEFAULT" | "FILE" | "MEMORY";
|
|
7
|
+
export type SQLitePragmaOptions = {
|
|
8
|
+
synchronous?: SQLiteSynchronousMode;
|
|
9
|
+
lockingMode?: SQLiteLockingMode;
|
|
10
|
+
tempStore?: SQLiteTempStoreMode;
|
|
11
|
+
};
|
|
12
|
+
|
|
3
13
|
interface Message {
|
|
4
14
|
id: string;
|
|
5
15
|
databaseId: string;
|
|
16
|
+
profile?: boolean;
|
|
17
|
+
protocol?: SqliteWorkerProtocol;
|
|
6
18
|
}
|
|
7
19
|
|
|
8
20
|
// Database messages
|
|
9
21
|
interface CreateDatabase extends Message {
|
|
10
22
|
type: "create";
|
|
11
23
|
directory?: string;
|
|
24
|
+
pragmas?: SQLitePragmaOptions;
|
|
12
25
|
}
|
|
13
26
|
|
|
14
27
|
interface Exec extends Message {
|
|
@@ -37,23 +50,102 @@ interface Prepare extends Message {
|
|
|
37
50
|
sql: string;
|
|
38
51
|
}
|
|
39
52
|
|
|
40
|
-
type
|
|
53
|
+
type Uint8ArrayBase64Type = {
|
|
54
|
+
type: "uint8array";
|
|
55
|
+
encoding: "base64";
|
|
56
|
+
base64: string;
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
type Uint8ArrayCloneType = {
|
|
60
|
+
type: "uint8array";
|
|
61
|
+
encoding: "clone";
|
|
62
|
+
value: Uint8Array;
|
|
63
|
+
};
|
|
41
64
|
|
|
42
65
|
type SimpleType = { type: "simple"; value: any };
|
|
43
66
|
|
|
44
|
-
export
|
|
45
|
-
|
|
46
|
-
|
|
67
|
+
export type EncodedValue =
|
|
68
|
+
| Uint8ArrayBase64Type
|
|
69
|
+
| Uint8ArrayCloneType
|
|
70
|
+
| SimpleType;
|
|
71
|
+
|
|
72
|
+
export type ClientEncodeMetrics = {
|
|
73
|
+
encodeMs: number;
|
|
74
|
+
valueCount: number;
|
|
75
|
+
blobValueCount: number;
|
|
76
|
+
blobBytes: number;
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
export type WorkerTiming = {
|
|
80
|
+
decodeMs: number;
|
|
81
|
+
execMs: number;
|
|
82
|
+
totalMs: number;
|
|
83
|
+
valueCount: number;
|
|
84
|
+
blobValueCount: number;
|
|
85
|
+
blobBytes: number;
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
export const resolveValue = (value: EncodedValue) => {
|
|
89
|
+
if (value.type === "simple") {
|
|
90
|
+
return value.value;
|
|
91
|
+
}
|
|
92
|
+
return value.encoding === "clone" ? value.value : fromBase64(value.base64);
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
export const encodeValue = (
|
|
96
|
+
value: any,
|
|
97
|
+
protocol: SqliteWorkerProtocol = "legacy",
|
|
98
|
+
): EncodedValue => {
|
|
47
99
|
if (value instanceof Uint8Array) {
|
|
48
|
-
return
|
|
100
|
+
return protocol === "clone"
|
|
101
|
+
? { type: "uint8array", encoding: "clone", value }
|
|
102
|
+
: { type: "uint8array", encoding: "base64", base64: toBase64(value) };
|
|
49
103
|
}
|
|
50
104
|
return { type: "simple", value };
|
|
51
105
|
};
|
|
52
106
|
|
|
107
|
+
export const encodeValues = (
|
|
108
|
+
values: any[] | undefined,
|
|
109
|
+
protocol: SqliteWorkerProtocol = "legacy",
|
|
110
|
+
): { values: EncodedValue[] | undefined; metrics: ClientEncodeMetrics } => {
|
|
111
|
+
if (!values || values.length === 0) {
|
|
112
|
+
return {
|
|
113
|
+
values,
|
|
114
|
+
metrics: {
|
|
115
|
+
encodeMs: 0,
|
|
116
|
+
valueCount: 0,
|
|
117
|
+
blobValueCount: 0,
|
|
118
|
+
blobBytes: 0,
|
|
119
|
+
},
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
let blobBytes = 0;
|
|
124
|
+
let blobValueCount = 0;
|
|
125
|
+
const startedAt = performance.now();
|
|
126
|
+
const encodedValues = values.map((value) => {
|
|
127
|
+
if (value instanceof Uint8Array) {
|
|
128
|
+
blobValueCount++;
|
|
129
|
+
blobBytes += value.byteLength;
|
|
130
|
+
}
|
|
131
|
+
return encodeValue(value, protocol);
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
return {
|
|
135
|
+
values: encodedValues,
|
|
136
|
+
metrics: {
|
|
137
|
+
encodeMs: performance.now() - startedAt,
|
|
138
|
+
valueCount: values.length,
|
|
139
|
+
blobValueCount,
|
|
140
|
+
blobBytes,
|
|
141
|
+
},
|
|
142
|
+
};
|
|
143
|
+
};
|
|
144
|
+
|
|
53
145
|
interface Run extends Statement {
|
|
54
146
|
type: "run";
|
|
55
147
|
sql: string;
|
|
56
|
-
values:
|
|
148
|
+
values: EncodedValue[];
|
|
57
149
|
}
|
|
58
150
|
|
|
59
151
|
// Statement messages
|
|
@@ -63,7 +155,7 @@ interface Statement extends Message {
|
|
|
63
155
|
|
|
64
156
|
interface Bind extends Statement {
|
|
65
157
|
type: "bind";
|
|
66
|
-
values:
|
|
158
|
+
values: EncodedValue[];
|
|
67
159
|
}
|
|
68
160
|
|
|
69
161
|
interface Step extends Statement {
|
|
@@ -72,7 +164,7 @@ interface Step extends Statement {
|
|
|
72
164
|
|
|
73
165
|
interface Get extends Statement {
|
|
74
166
|
type: "get";
|
|
75
|
-
values?:
|
|
167
|
+
values?: EncodedValue[];
|
|
76
168
|
}
|
|
77
169
|
|
|
78
170
|
interface Reset extends Statement {
|
|
@@ -81,12 +173,12 @@ interface Reset extends Statement {
|
|
|
81
173
|
|
|
82
174
|
interface RunStatement extends Statement {
|
|
83
175
|
type: "run-statement";
|
|
84
|
-
values:
|
|
176
|
+
values: EncodedValue[];
|
|
85
177
|
}
|
|
86
178
|
|
|
87
179
|
interface All extends Statement {
|
|
88
180
|
type: "all";
|
|
89
|
-
values:
|
|
181
|
+
values: EncodedValue[];
|
|
90
182
|
}
|
|
91
183
|
|
|
92
184
|
interface Finalize extends Statement {
|
|
@@ -98,12 +190,14 @@ interface ErrorResponse {
|
|
|
98
190
|
type: "error";
|
|
99
191
|
id: string;
|
|
100
192
|
message: string;
|
|
193
|
+
timing?: WorkerTiming;
|
|
101
194
|
}
|
|
102
195
|
|
|
103
196
|
interface Response {
|
|
104
197
|
type: "response";
|
|
105
198
|
id: string;
|
|
106
199
|
result: any;
|
|
200
|
+
timing?: WorkerTiming;
|
|
107
201
|
}
|
|
108
202
|
|
|
109
203
|
export type DatabaseMessages =
|