@peerbit/indexer-sqlite3 3.0.0-e6ea5c0 → 3.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/dist/assets/sqlite3/sqlite3.worker.min.js +168 -52
  2. package/dist/index.min.js +277 -53
  3. package/dist/index.min.js.map +3 -3
  4. package/dist/src/engine.d.ts +9 -3
  5. package/dist/src/engine.d.ts.map +1 -1
  6. package/dist/src/engine.js +75 -9
  7. package/dist/src/engine.js.map +1 -1
  8. package/dist/src/index.d.ts +7 -3
  9. package/dist/src/index.d.ts.map +1 -1
  10. package/dist/src/index.js +5 -4
  11. package/dist/src/index.js.map +1 -1
  12. package/dist/src/schema.d.ts +5 -1
  13. package/dist/src/schema.d.ts.map +1 -1
  14. package/dist/src/schema.js +42 -12
  15. package/dist/src/schema.js.map +1 -1
  16. package/dist/src/sqlite3-messages.worker.d.ts +47 -8
  17. package/dist/src/sqlite3-messages.worker.d.ts.map +1 -1
  18. package/dist/src/sqlite3-messages.worker.js +42 -3
  19. package/dist/src/sqlite3-messages.worker.js.map +1 -1
  20. package/dist/src/sqlite3.browser.d.ts +22 -1
  21. package/dist/src/sqlite3.browser.d.ts.map +1 -1
  22. package/dist/src/sqlite3.browser.js +138 -31
  23. package/dist/src/sqlite3.browser.js.map +1 -1
  24. package/dist/src/sqlite3.d.ts +4 -1
  25. package/dist/src/sqlite3.d.ts.map +1 -1
  26. package/dist/src/sqlite3.js +14 -5
  27. package/dist/src/sqlite3.js.map +1 -1
  28. package/dist/src/sqlite3.wasm.d.ts +4 -1
  29. package/dist/src/sqlite3.wasm.d.ts.map +1 -1
  30. package/dist/src/sqlite3.wasm.js +15 -3
  31. package/dist/src/sqlite3.wasm.js.map +1 -1
  32. package/dist/src/sqlite3.worker.js +146 -53
  33. package/dist/src/sqlite3.worker.js.map +1 -1
  34. package/dist/src/utils.d.ts +1 -0
  35. package/dist/src/utils.d.ts.map +1 -1
  36. package/dist/src/utils.js +11 -0
  37. package/dist/src/utils.js.map +1 -1
  38. package/package.json +10 -9
  39. package/src/engine.ts +99 -11
  40. package/src/index.ts +39 -4
  41. package/src/schema.ts +67 -11
  42. package/src/sqlite3-messages.worker.ts +104 -10
  43. package/src/sqlite3.browser.ts +246 -88
  44. package/src/sqlite3.ts +19 -5
  45. package/src/sqlite3.wasm.ts +25 -3
  46. package/src/sqlite3.worker.ts +170 -49
  47. package/src/utils.ts +14 -0
package/src/schema.ts CHANGED
@@ -817,7 +817,11 @@ const isUint8ArrayType = (type: FieldType) => {
817
817
  };
818
818
 
819
819
  export const insert = async (
820
- insertFn: (values: any[], table: Table) => Promise<any> | any,
820
+ insertFn: (
821
+ values: any[],
822
+ table: Table,
823
+ options?: { requireId?: boolean },
824
+ ) => Promise<any> | any,
821
825
  obj: Record<string, any>,
822
826
  tables: Map<string, Table>,
823
827
  table: Table,
@@ -827,6 +831,12 @@ export const insert = async (
827
831
  ) => Promise<void> | void | number,
828
832
  parentId: any = undefined,
829
833
  index?: number,
834
+ options?: {
835
+ insertSimpleVecRows?: (
836
+ rows: any[][],
837
+ table: Table,
838
+ ) => Promise<void> | void;
839
+ },
830
840
  ): Promise<void> => {
831
841
  const bindableValues: any[] = [];
832
842
  let nestedCallbacks: ((id: any) => Promise<void>)[] = [];
@@ -836,6 +846,14 @@ export const insert = async (
836
846
  ? handleNestedCallback
837
847
  : (fn) => nestedCallbacks.push(fn);
838
848
 
849
+ const toInsertValue = (item: any, subTable: Table) =>
850
+ typeof item === "function" && item instanceof Uint8Array === false
851
+ ? item
852
+ : subTable.isSimpleValue
853
+ ? // eslint-disable-next-line new-cap
854
+ new subTable.ctor(item)
855
+ : Object.assign(Object.create(subTable.ctor.prototype), item);
856
+
839
857
  const handleElement = async (
840
858
  item: any,
841
859
  field: Field,
@@ -846,18 +864,14 @@ export const insert = async (
846
864
 
847
865
  await insert(
848
866
  insertFn,
849
- typeof item === "function" && item instanceof Uint8Array === false
850
- ? item
851
- : subTable.isSimpleValue
852
- ? // eslint-disable-next-line new-cap
853
- new subTable.ctor(item)
854
- : Object.assign(Object.create(subTable.ctor.prototype), item),
867
+ toInsertValue(item, subTable),
855
868
  tables,
856
869
  subTable,
857
870
  getSchema(subTable.ctor).fields,
858
871
  handleNestedCallback,
859
872
  parentId,
860
873
  index,
874
+ options,
861
875
  );
862
876
  };
863
877
 
@@ -868,6 +882,39 @@ export const insert = async (
868
882
  ) => {
869
883
  if (Array.isArray(obj[field.key])) {
870
884
  const arr = obj[field.key];
885
+ const firstItem = arr.find((item: any) => item != null);
886
+ if (
887
+ parentId != null &&
888
+ arr.length > 1 &&
889
+ firstItem != null &&
890
+ options?.insertSimpleVecRows
891
+ ) {
892
+ const subTable = getTableFromValue(table, tables, field, firstItem);
893
+ if (subTable.isSimpleValue) {
894
+ const rows: any[][] = [];
895
+ for (let i = 0; i < arr.length; i++) {
896
+ const item = arr[i];
897
+ await insert(
898
+ (values) => {
899
+ rows.push(values);
900
+ return undefined;
901
+ },
902
+ toInsertValue(item, subTable),
903
+ tables,
904
+ subTable,
905
+ getSchema(subTable.ctor).fields,
906
+ handleNestedCallback,
907
+ parentId,
908
+ i,
909
+ options,
910
+ );
911
+ }
912
+ if (rows.length > 0) {
913
+ await options.insertSimpleVecRows(rows, subTable);
914
+ }
915
+ return;
916
+ }
917
+ }
871
918
  for (let i = 0; i < arr.length; i++) {
872
919
  const item = arr[i];
873
920
  await handleElement(item, field, parentId, i);
@@ -945,6 +992,7 @@ export const insert = async (
945
992
  (fn) => nestedCallbacks.push(fn),
946
993
  undefined, // parentId is not defined here, we are inserting a nested object inline
947
994
  undefined, // index is not defined here, we are inserting a nested object inline
995
+ options,
948
996
  );
949
997
  /* await insert(, obj[field.key], tables, subTable, getSchema(unwrappedType).fields, parentId, index); */
950
998
  } else {
@@ -963,7 +1011,9 @@ export const insert = async (
963
1011
  await handleNestedCallback!((id) => handleNested(nested, isOptional, id));
964
1012
  }
965
1013
 
966
- const thisId = await insertFn(bindableValues, table);
1014
+ const thisId = await insertFn(bindableValues, table, {
1015
+ requireId: nestedCallbacks.length > 0,
1016
+ });
967
1017
  if (table.primary === false && nestedCallbacks.length > 0) {
968
1018
  throw new Error("Unexpected");
969
1019
  }
@@ -1875,9 +1925,15 @@ const _buildJoin = (
1875
1925
 
1876
1926
  if (table!.columns.length > 0) {
1877
1927
  const usedColumns = removeDuplicatesOrdered(table!.columns);
1878
- indexedBy = options?.planner
1879
- ? ` INDEXED BY ${options.planner.resolveIndex(table.table.name, usedColumns)} `
1880
- : "";
1928
+ const usesImplicitPrimaryKeyIndex =
1929
+ table.type === "root" &&
1930
+ table.table.primary !== false &&
1931
+ usedColumns.length === 1 &&
1932
+ usedColumns[0] === table.table.primary;
1933
+ indexedBy =
1934
+ options?.planner && !usesImplicitPrimaryKeyIndex
1935
+ ? ` INDEXED BY ${options.planner.resolveIndex(table.table.name, usedColumns)} `
1936
+ : "";
1881
1937
  }
1882
1938
 
1883
1939
  if (table.type !== "root") {
@@ -1,14 +1,27 @@
1
1
  import { fromBase64, toBase64 } from "@peerbit/crypto";
2
2
 
3
+ export type SqliteWorkerProtocol = "legacy" | "clone";
4
+ export type SQLiteSynchronousMode = "FULL" | "NORMAL" | "OFF";
5
+ export type SQLiteLockingMode = "NORMAL" | "EXCLUSIVE";
6
+ export type SQLiteTempStoreMode = "DEFAULT" | "FILE" | "MEMORY";
7
+ export type SQLitePragmaOptions = {
8
+ synchronous?: SQLiteSynchronousMode;
9
+ lockingMode?: SQLiteLockingMode;
10
+ tempStore?: SQLiteTempStoreMode;
11
+ };
12
+
3
13
  interface Message {
4
14
  id: string;
5
15
  databaseId: string;
16
+ profile?: boolean;
17
+ protocol?: SqliteWorkerProtocol;
6
18
  }
7
19
 
8
20
  // Database messages
9
21
  interface CreateDatabase extends Message {
10
22
  type: "create";
11
23
  directory?: string;
24
+ pragmas?: SQLitePragmaOptions;
12
25
  }
13
26
 
14
27
  interface Exec extends Message {
@@ -37,23 +50,102 @@ interface Prepare extends Message {
37
50
  sql: string;
38
51
  }
39
52
 
40
- type Uint8ArrayType = { type: "uint8array"; base64: string };
53
+ type Uint8ArrayBase64Type = {
54
+ type: "uint8array";
55
+ encoding: "base64";
56
+ base64: string;
57
+ };
58
+
59
+ type Uint8ArrayCloneType = {
60
+ type: "uint8array";
61
+ encoding: "clone";
62
+ value: Uint8Array;
63
+ };
41
64
 
42
65
  type SimpleType = { type: "simple"; value: any };
43
66
 
44
- export const resolveValue = (value: Uint8ArrayType | SimpleType) =>
45
- value.type === "simple" ? value.value : fromBase64(value.base64);
46
- export const encodeValue = (value: any): Uint8ArrayType | SimpleType => {
67
+ export type EncodedValue =
68
+ | Uint8ArrayBase64Type
69
+ | Uint8ArrayCloneType
70
+ | SimpleType;
71
+
72
+ export type ClientEncodeMetrics = {
73
+ encodeMs: number;
74
+ valueCount: number;
75
+ blobValueCount: number;
76
+ blobBytes: number;
77
+ };
78
+
79
+ export type WorkerTiming = {
80
+ decodeMs: number;
81
+ execMs: number;
82
+ totalMs: number;
83
+ valueCount: number;
84
+ blobValueCount: number;
85
+ blobBytes: number;
86
+ };
87
+
88
+ export const resolveValue = (value: EncodedValue) => {
89
+ if (value.type === "simple") {
90
+ return value.value;
91
+ }
92
+ return value.encoding === "clone" ? value.value : fromBase64(value.base64);
93
+ };
94
+
95
+ export const encodeValue = (
96
+ value: any,
97
+ protocol: SqliteWorkerProtocol = "legacy",
98
+ ): EncodedValue => {
47
99
  if (value instanceof Uint8Array) {
48
- return { type: "uint8array", base64: toBase64(value) };
100
+ return protocol === "clone"
101
+ ? { type: "uint8array", encoding: "clone", value }
102
+ : { type: "uint8array", encoding: "base64", base64: toBase64(value) };
49
103
  }
50
104
  return { type: "simple", value };
51
105
  };
52
106
 
107
+ export const encodeValues = (
108
+ values: any[] | undefined,
109
+ protocol: SqliteWorkerProtocol = "legacy",
110
+ ): { values: EncodedValue[] | undefined; metrics: ClientEncodeMetrics } => {
111
+ if (!values || values.length === 0) {
112
+ return {
113
+ values,
114
+ metrics: {
115
+ encodeMs: 0,
116
+ valueCount: 0,
117
+ blobValueCount: 0,
118
+ blobBytes: 0,
119
+ },
120
+ };
121
+ }
122
+
123
+ let blobBytes = 0;
124
+ let blobValueCount = 0;
125
+ const startedAt = performance.now();
126
+ const encodedValues = values.map((value) => {
127
+ if (value instanceof Uint8Array) {
128
+ blobValueCount++;
129
+ blobBytes += value.byteLength;
130
+ }
131
+ return encodeValue(value, protocol);
132
+ });
133
+
134
+ return {
135
+ values: encodedValues,
136
+ metrics: {
137
+ encodeMs: performance.now() - startedAt,
138
+ valueCount: values.length,
139
+ blobValueCount,
140
+ blobBytes,
141
+ },
142
+ };
143
+ };
144
+
53
145
  interface Run extends Statement {
54
146
  type: "run";
55
147
  sql: string;
56
- values: (Uint8ArrayType | SimpleType)[];
148
+ values: EncodedValue[];
57
149
  }
58
150
 
59
151
  // Statement messages
@@ -63,7 +155,7 @@ interface Statement extends Message {
63
155
 
64
156
  interface Bind extends Statement {
65
157
  type: "bind";
66
- values: (Uint8ArrayType | SimpleType)[];
158
+ values: EncodedValue[];
67
159
  }
68
160
 
69
161
  interface Step extends Statement {
@@ -72,7 +164,7 @@ interface Step extends Statement {
72
164
 
73
165
  interface Get extends Statement {
74
166
  type: "get";
75
- values?: any[];
167
+ values?: EncodedValue[];
76
168
  }
77
169
 
78
170
  interface Reset extends Statement {
@@ -81,12 +173,12 @@ interface Reset extends Statement {
81
173
 
82
174
  interface RunStatement extends Statement {
83
175
  type: "run-statement";
84
- values: any[];
176
+ values: EncodedValue[];
85
177
  }
86
178
 
87
179
  interface All extends Statement {
88
180
  type: "all";
89
- values: (Uint8ArrayType | SimpleType)[];
181
+ values: EncodedValue[];
90
182
  }
91
183
 
92
184
  interface Finalize extends Statement {
@@ -98,12 +190,14 @@ interface ErrorResponse {
98
190
  type: "error";
99
191
  id: string;
100
192
  message: string;
193
+ timing?: WorkerTiming;
101
194
  }
102
195
 
103
196
  interface Response {
104
197
  type: "response";
105
198
  id: string;
106
199
  result: any;
200
+ timing?: WorkerTiming;
107
201
  }
108
202
 
109
203
  export type DatabaseMessages =