@docstack/pouchdb-adapter-googledrive 0.0.8 → 0.0.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. package/compaction_test.log +725 -0
  2. package/docs/DEBUG_BIDIRECTIONAL_REPLICATION.ipynb +481 -0
  3. package/docs/DOCUMENTATION.md +115 -0
  4. package/docs/PROMISE_SUPPORT.md +75 -0
  5. package/docs/REPLICATION_ISSUE.md +63 -0
  6. package/emulation-test.log +163 -0
  7. package/failing_test.log +31 -0
  8. package/final_test.log +2620 -0
  9. package/lib/adapter.d.ts +20 -0
  10. package/lib/adapter.js +650 -0
  11. package/lib/cache.d.ts +12 -0
  12. package/lib/cache.js +42 -0
  13. package/lib/client.d.ts +37 -0
  14. package/lib/client.js +189 -0
  15. package/lib/drive.d.ts +82 -0
  16. package/lib/drive.js +737 -0
  17. package/lib/index.d.ts +10 -0
  18. package/lib/index.js +56 -0
  19. package/lib/types.d.ts +92 -0
  20. package/lib/types.js +2 -0
  21. package/live_test.log +1209 -0
  22. package/package.json +1 -1
  23. package/test_output.log +563 -0
  24. package/test_output_2.log +1408 -0
  25. package/test_output_3.log +2445 -0
  26. package/.test-drive-integration/1xssnxhy3akj +0 -1
  27. package/.test-drive-integration/8p8eymf8afm +0 -1
  28. package/.test-drive-integration/d1a4udbbuh +0 -1
  29. package/.test-drive-integration/usx93j46xtl +0 -1
  30. package/.test-drive-integration/vc0pq3g8vp +0 -1
  31. package/.test-drive-integration/zenra4v6i +0 -1
  32. package/.test-drive-root/05wzn5mtlrzt +0 -1
  33. package/.test-drive-root/0axnnh1gmn2d +0 -1
  34. package/.test-drive-root/0l2pw73vk4i +0 -1
  35. package/.test-drive-root/0tzush8sunaj +0 -1
  36. package/.test-drive-root/143bh2d29rnl +0 -1
  37. package/.test-drive-root/1synnvu59il +0 -1
  38. package/.test-drive-root/239cd47eh23i +0 -1
  39. package/.test-drive-root/275ocoaj6nq +0 -1
  40. package/.test-drive-root/29d78af16s4 +0 -1
  41. package/.test-drive-root/2gzvpzclnwu +0 -1
  42. package/.test-drive-root/2pmk0leotvp +0 -1
  43. package/.test-drive-root/38ijn3s7p33 +0 -1
  44. package/.test-drive-root/3jm64la13rp +0 -1
  45. package/.test-drive-root/3l4zrzi5bm2 +0 -1
  46. package/.test-drive-root/3of67xqpsfh +0 -1
  47. package/.test-drive-root/3pi1rk70we5 +0 -1
  48. package/.test-drive-root/3w3n7tpdn4n +0 -1
  49. package/.test-drive-root/3xg31irodim +0 -1
  50. package/.test-drive-root/50asm455hsa +0 -1
  51. package/.test-drive-root/51ldbder4dm +0 -1
  52. package/.test-drive-root/5l7dt9r491r +0 -1
  53. package/.test-drive-root/65mfhb9yuq5 +0 -1
  54. package/.test-drive-root/6iocm2rs118 +0 -1
  55. package/.test-drive-root/6o6vlpa6r6j +0 -1
  56. package/.test-drive-root/6rrcl4sdhoi +0 -1
  57. package/.test-drive-root/6ucq10yu9bx +0 -1
  58. package/.test-drive-root/7de4gtsch5b +0 -1
  59. package/.test-drive-root/7j4j3eq54q +0 -1
  60. package/.test-drive-root/7n3ai8wgsvy +0 -1
  61. package/.test-drive-root/7v4b3eyitw3 +0 -1
  62. package/.test-drive-root/8e15eu4xksq +0 -1
  63. package/.test-drive-root/8pu2c63yh1j +0 -1
  64. package/.test-drive-root/8ryxg058q8g +0 -1
  65. package/.test-drive-root/99l00ber08g +0 -1
  66. package/.test-drive-root/9gxcumnvepb +0 -1
  67. package/.test-drive-root/a4xxhwpe2cp +0 -1
  68. package/.test-drive-root/a57knn0ufon +0 -1
  69. package/.test-drive-root/a7gszl8y5ir +0 -1
  70. package/.test-drive-root/b0nddw4fdbu +0 -1
  71. package/.test-drive-root/b3b1d1yuolh +0 -1
  72. package/.test-drive-root/b4syq7nig1f +0 -1
  73. package/.test-drive-root/bcc8mhlffkg +0 -1
  74. package/.test-drive-root/bmngbpsk34 +0 -1
  75. package/.test-drive-root/c9i4hvcll2s +0 -1
  76. package/.test-drive-root/cao688yanw +0 -1
  77. package/.test-drive-root/cdtduwukrek +0 -1
  78. package/.test-drive-root/ckzeli8w7ej +0 -1
  79. package/.test-drive-root/ctywezu2fi5 +0 -1
  80. package/.test-drive-root/cyxiv5vgtle +0 -1
  81. package/.test-drive-root/d83g0suoz08 +0 -1
  82. package/.test-drive-root/d9tefhps5m6 +0 -1
  83. package/.test-drive-root/dac156l068 +0 -1
  84. package/.test-drive-root/daqsvyj5cu9 +0 -1
  85. package/.test-drive-root/df0wbnhhz1 +0 -1
  86. package/.test-drive-root/difkbp1duv +0 -1
  87. package/.test-drive-root/djrry1wk98v +0 -1
  88. package/.test-drive-root/dx42jrz7ty9 +0 -1
  89. package/.test-drive-root/e0s76kp3ena +0 -1
  90. package/.test-drive-root/e9xvdv0mqig +0 -1
  91. package/.test-drive-root/f3xo9hy1956 +0 -1
  92. package/.test-drive-root/f4scgrpgejw +0 -1
  93. package/.test-drive-root/f9kbt63p7xp +0 -1
  94. package/.test-drive-root/fe07yvs6p14 +0 -1
  95. package/.test-drive-root/ff5suiec8xv +0 -1
  96. package/.test-drive-root/fwc1qm37as +0 -1
  97. package/.test-drive-root/gh39v4lzxjp +0 -1
  98. package/.test-drive-root/gxvkr1qq21b +0 -1
  99. package/.test-drive-root/h4p64z7djk5 +0 -1
  100. package/.test-drive-root/habe7525tnl +0 -1
  101. package/.test-drive-root/hhzddxj31ar +0 -1
  102. package/.test-drive-root/hyj9crn9sb +0 -1
  103. package/.test-drive-root/igey5y5f3mi +0 -1
  104. package/.test-drive-root/isvxwzkgym +0 -1
  105. package/.test-drive-root/javyqnt07ws +0 -1
  106. package/.test-drive-root/jevlhmyaczl +0 -1
  107. package/.test-drive-root/jve7ypin7bb +0 -1
  108. package/.test-drive-root/koycie6jmyr +0 -1
  109. package/.test-drive-root/kvlad8ps1l +0 -1
  110. package/.test-drive-root/l13wzs51qi +0 -1
  111. package/.test-drive-root/lbwwk07et6i +0 -1
  112. package/.test-drive-root/m3uoxcsft2g +0 -1
  113. package/.test-drive-root/m7l8z61tnei +0 -1
  114. package/.test-drive-root/mmwpyz6aa2e +0 -1
  115. package/.test-drive-root/mqbyniatwu +0 -1
  116. package/.test-drive-root/mz3jbwwr12 +0 -1
  117. package/.test-drive-root/mza04oxze7 +0 -1
  118. package/.test-drive-root/ndqlafk0c0c +0 -1
  119. package/.test-drive-root/noojz8gst2 +0 -1
  120. package/.test-drive-root/o8ujk9otq7 +0 -1
  121. package/.test-drive-root/oa54zbip13m +0 -1
  122. package/.test-drive-root/ore7c15j64a +0 -1
  123. package/.test-drive-root/oxj3tt9q16 +0 -1
  124. package/.test-drive-root/pac4f4x1r9 +0 -1
  125. package/.test-drive-root/pqawj7k06i +0 -1
  126. package/.test-drive-root/qujzt0hpj6i +0 -1
  127. package/.test-drive-root/r5sdb4w0d3 +0 -1
  128. package/.test-drive-root/r8pybcnme1r +0 -1
  129. package/.test-drive-root/rpht4r0nk9 +0 -1
  130. package/.test-drive-root/rru3wcrfved +0 -1
  131. package/.test-drive-root/rtq2tifpvgt +0 -1
  132. package/.test-drive-root/ru00ubalaar +0 -1
  133. package/.test-drive-root/rx65wqybqlm +0 -1
  134. package/.test-drive-root/s585nksqc5 +0 -1
  135. package/.test-drive-root/scvyemdb9gh +0 -1
  136. package/.test-drive-root/si2b3ac0lve +0 -1
  137. package/.test-drive-root/sjbanmc5v5k +0 -1
  138. package/.test-drive-root/sl2coirs8zo +0 -1
  139. package/.test-drive-root/so3w2nrxm7 +0 -1
  140. package/.test-drive-root/t3y2vj69hz9 +0 -1
  141. package/.test-drive-root/t95zgk2fl8f +0 -1
  142. package/.test-drive-root/t9f08z6ayj +0 -1
  143. package/.test-drive-root/ta501uh71xi +0 -1
  144. package/.test-drive-root/taqhm9suyu +0 -1
  145. package/.test-drive-root/tya2ijynx5k +0 -1
  146. package/.test-drive-root/u4llyu4o7en +0 -1
  147. package/.test-drive-root/u5wco26a6lb +0 -1
  148. package/.test-drive-root/u6sricrto +0 -1
  149. package/.test-drive-root/ukcmmzglxa +0 -1
  150. package/.test-drive-root/uvsiktw52p +0 -1
  151. package/.test-drive-root/uylv1bn9swb +0 -1
  152. package/.test-drive-root/uzkzgxo3k5b +0 -1
  153. package/.test-drive-root/vf68w1pfrv +0 -1
  154. package/.test-drive-root/vp4g881bu7 +0 -1
  155. package/.test-drive-root/vuv8aavujb +0 -1
  156. package/.test-drive-root/wg24h3o6df +0 -1
  157. package/.test-drive-root/wyi5oftc8nh +0 -1
  158. package/.test-drive-root/x5zl3x2eh +0 -1
  159. package/.test-drive-root/xb3bpq9xtk +0 -1
  160. package/.test-drive-root/xfn089pthf9 +0 -1
  161. package/.test-drive-root/xrs7jul0mdl +0 -1
  162. package/.test-drive-root/y5qtilrbuzq +0 -1
  163. package/.test-drive-root/ysxlnyjuav +0 -1
  164. package/.test-drive-root/z2loakinuyk +0 -1
  165. package/.test-drive-root/z6lxmw8opjo +0 -1
  166. package/.test-drive-root/z74cnyignf9 +0 -1
  167. package/.test-drive-root/zi9agjhrfw +0 -1
  168. package/.test-drive-root/zspmm1wmo5l +0 -1
package/lib/drive.js ADDED
@@ -0,0 +1,737 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.DriveHandler = void 0;
4
+ const cache_1 = require("./cache");
5
+ const client_1 = require("./client");
6
+ const DEFAULT_COMPACTION_THRESHOLD = 100; // entries
7
+ const DEFAULT_SIZE_THRESHOLD = 1024 * 1024; // 1MB
8
+ const DEFAULT_CACHE_SIZE = 1000; // Number of docs
9
+ /**
10
+ * DriveHandler - Lazy Loading Implementation
11
+ *
12
+ * Storage structure:
13
+ * /db-folder/
14
+ * ├── _meta.json
15
+ * ├── snapshot-index.json # Map<DocId, IndexEntry>
16
+ * ├── snapshot-data.json # Map<DocId, DocBody>
17
+ * └── changes-*.ndjson # Append logs
18
+ */
19
+ class DriveHandler {
20
+ log(...args) {
21
+ console.log(`[googledrive-drive] [${this.meta.dbName}]`, ...args);
22
+ }
23
+ constructor(options, dbName) {
24
+ this.folderId = null;
25
+ this.meta = {
26
+ seq: 0,
27
+ changeLogIds: [],
28
+ snapshotIndexId: null,
29
+ lastCompaction: null,
30
+ dbName: ''
31
+ };
32
+ this.metaEtag = null;
33
+ this.metaModifiedTime = null;
34
+ // In-Memory Index: ID -> Metadata/Pointer
35
+ this.index = {};
36
+ this.pendingChanges = [];
37
+ this.currentLogSizeEstimate = 0;
38
+ this.listeners = [];
39
+ this.pollingInterval = null;
40
+ this.loadingPromise = null;
41
+ this.isPollingActive = false;
42
+ this.processedLogIds = new Set();
43
+ this.currentSnapshotIndexId = null;
44
+ this.debug = false;
45
+ this.isCompacting = false;
46
+ const clientOptions = { ...options };
47
+ if (options.testMode) {
48
+ const serverUrl = options.testServerUrl || 'http://localhost:3000';
49
+ // @ts-ignore - baseUrl/uploadUrl might not be in the strict type if we didn't update types.ts definition for DriveClientOptions in client.ts yet?
50
+ // We did update DriveClientOptions in client.ts.
51
+ // But GoogleDriveAdapterOptions extends DriveClientOptions?
52
+ // types.ts: export interface GoogleDriveAdapterOptions extends DriveClientOptions
53
+ // client.ts: export interface DriveClientOptions { accessToken: ...; baseUrl?: string; uploadUrl?: string; }
54
+ // So typescript should be happy.
55
+ // Using /drive/v3/files as base for the test server if simplified?
56
+ // The TestServer mounts at /drive/v3/files.
57
+ // But the client appends /drive/v3/files to BASE_URL?
58
+ // In client.ts default is `https://www.googleapis.com/drive/v3/files`.
59
+ // Our TestServer mounts `/drive/v3/files`.
60
+ // So testUrl should be `http://localhost:3000/drive/v3/files`.
61
+ const testBase = `${serverUrl}/drive/v3/files`;
62
+ const testUpload = `${serverUrl}/upload/drive/v3/files`;
63
+ // @ts-ignore
64
+ clientOptions.baseUrl = testBase;
65
+ // @ts-ignore
66
+ clientOptions.uploadUrl = testUpload;
67
+ }
68
+ this.client = new client_1.GoogleDriveClient(clientOptions);
69
+ this.options = options;
70
+ this.folderId = options.folderId || null;
71
+ this.folderName = options.folderName || dbName;
72
+ this.parents = options.parents || [];
73
+ this.compactionThreshold = options.compactionThreshold || DEFAULT_COMPACTION_THRESHOLD;
74
+ this.compactionSizeThreshold = options.compactionSizeThreshold || DEFAULT_SIZE_THRESHOLD;
75
+ this.meta.dbName = dbName;
76
+ this.debug = !!options.debug;
77
+ this.docCache = new cache_1.LRUCache(options.cacheSize || DEFAULT_CACHE_SIZE);
78
+ this.fileCache = new cache_1.LRUCache(100); // Cache for last 100 files
79
+ // Polling will be started in load() after folderId is resolved
80
+ }
81
+ // Public getter for Sequence (used by adapter)
82
+ get seq() {
83
+ return this.meta.seq;
84
+ }
85
+ /** Load the database (Index Only) */
86
+ async load() {
87
+ if (this.loadingPromise)
88
+ return this.loadingPromise;
89
+ this.loadingPromise = (async () => {
90
+ try {
91
+ this.log('Loading database, options', { options: this.options });
92
+ if (!this.folderId) {
93
+ this.folderId = await this.findOrCreateFolder();
94
+ this.log('Retrieved folder', { folderId: this.folderId });
95
+ }
96
+ const metaFile = await this.findFile('_meta.json');
97
+ if (metaFile) {
98
+ this.log('Retrieved meta file', { fileId: metaFile.id });
99
+ this.meta = await this.downloadJson(metaFile.id, true); // No cache for meta
100
+ this.metaEtag = metaFile.etag || null;
101
+ this.metaModifiedTime = metaFile.modifiedTime || null;
102
+ }
103
+ else {
104
+ this.log('Meta file not found, creating new');
105
+ await this.saveMeta(this.meta);
106
+ }
107
+ if (this.meta.snapshotIndexId !== this.currentSnapshotIndexId) {
108
+ this.log('Snapshot index changed, loading index', {
109
+ snapshotIndexId: this.meta.snapshotIndexId,
110
+ currentSnapshotIndexId: this.currentSnapshotIndexId
111
+ });
112
+ // Compaction occurred or initial load
113
+ this.index = {};
114
+ this.processedLogIds.clear();
115
+ this.currentSnapshotIndexId = this.meta.snapshotIndexId;
116
+ if (this.meta.snapshotIndexId) {
117
+ try {
118
+ const snapshotIdx = await this.downloadJson(this.meta.snapshotIndexId);
119
+ if (snapshotIdx.docs) {
120
+ this.filesFromLegacySnapshot(snapshotIdx);
121
+ }
122
+ else {
123
+ this.index = snapshotIdx.entries || {};
124
+ }
125
+ }
126
+ catch (e) {
127
+ console.warn('Failed to load snapshot index', e);
128
+ }
129
+ }
130
+ else if (this.meta.snapshotId) {
131
+ this.log('Legacy snapshot found, loading index', {
132
+ snapshotId: this.meta.snapshotId
133
+ });
134
+ try {
135
+ const legacySnapshot = await this.downloadJson(this.meta.snapshotId);
136
+ this.filesFromLegacySnapshot(legacySnapshot);
137
+ }
138
+ catch (e) {
139
+ console.warn('Failed to load legacy snapshot', e);
140
+ }
141
+ }
142
+ }
143
+ // 2. Replay NEW Change Logs (Metadata only updates)
144
+ this.log('Replaying change logs');
145
+ for (const logId of this.meta.changeLogIds) {
146
+ if (this.processedLogIds.has(logId))
147
+ continue;
148
+ this.log('Replaying change log', logId);
149
+ let changes = await this.downloadNdjson(logId);
150
+ if (!Array.isArray(changes)) {
151
+ this.log("Unexpected changes", { changes, logId });
152
+ this.log('Downloaded changes not an array, wrapping/ignoring', typeof changes);
153
+ changes = changes ? [changes] : []; // Fallback
154
+ }
155
+ this.currentLogSizeEstimate += 100 * changes.length;
156
+ this.log('Replayed change log', logId);
157
+ for (const change of changes) {
158
+ this.log('Processing change, sequence', change.seq);
159
+ this.updateIndex(change, logId);
160
+ if (this.docCache.get(change.id)) {
161
+ this.docCache.remove(change.id);
162
+ }
163
+ }
164
+ this.processedLogIds.add(logId);
165
+ this.log('Processed log', logId);
166
+ }
167
+ // 3. Start Polling (if enabled)
168
+ if (this.options.pollingIntervalMs) {
169
+ this.log('Starting polling with interval', this.options.pollingIntervalMs);
170
+ this.startPolling(Number(this.options.pollingIntervalMs));
171
+ }
172
+ else {
173
+ this.log('Polling disabled (no interval provided)');
174
+ }
175
+ }
176
+ catch (e) {
177
+ console.error('Failed to load database', e);
178
+ throw e;
179
+ }
180
+ finally {
181
+ this.loadingPromise = null;
182
+ }
183
+ })();
184
+ return this.loadingPromise;
185
+ }
186
+ // Migration helper
187
+ filesFromLegacySnapshot(snapshot) {
188
+ // Convert Legacy Snapshot (Docs in memory) to Index
189
+ // Since we don't have a separate file pointer for each doc in legacy snapshot,
190
+ // we say they are in the snapshot file itself.
191
+ // BUT, lazy loading requires being able to fetch them.
192
+ // We will cache them ALL now (since we downloaded them) and index them.
193
+ for (const [id, doc] of Object.entries(snapshot.docs)) {
194
+ this.index[id] = {
195
+ rev: doc._rev,
196
+ seq: snapshot.seq, // Approximate
197
+ location: { fileId: 'LEGACY_MEMORY' } // Special validity marker
198
+ };
199
+ this.docCache.put(id, doc);
200
+ }
201
+ }
202
+ /**
203
+ * Get a document body.
204
+ * Index -> Cache -> Fetch
205
+ */
206
+ async get(id) {
207
+ const entry = this.index[id];
208
+ if (!entry)
209
+ return null;
210
+ if (entry.deleted)
211
+ return null;
212
+ // 1. Check Doc Cache
213
+ const cachedDoc = this.docCache.get(id);
214
+ if (cachedDoc)
215
+ return cachedDoc;
216
+ // 2. Fetch from Drive (via File Cache)
217
+ if (entry.location.fileId === 'LEGACY_MEMORY') {
218
+ if (this.meta.snapshotId) {
219
+ const legacy = await this.downloadJson(this.meta.snapshotId);
220
+ if (legacy.docs[id]) {
221
+ this.docCache.put(id, legacy.docs[id]);
222
+ return legacy.docs[id];
223
+ }
224
+ }
225
+ return null;
226
+ }
227
+ const fileId = entry.location.fileId;
228
+ const content = await this.fetchFile(fileId);
229
+ let doc = null;
230
+ if (Array.isArray(content)) {
231
+ // It's a change log (NDJSON parsed as array)
232
+ const match = [...content].reverse().find((c) => c.id === id);
233
+ doc = match ? match.doc : null;
234
+ }
235
+ else if (content && content.docs) {
236
+ // It's a snapshot-data chunk
237
+ doc = content.docs[id];
238
+ }
239
+ else if (content && content.id === id && content.doc) {
240
+ // It's a single ChangeEntry object (parsed from single-line NDJSON)
241
+ doc = content.doc;
242
+ }
243
+ else if (content && (content._id === id || content.id === id)) {
244
+ // Single doc file or raw doc body
245
+ doc = content;
246
+ }
247
+ if (doc) {
248
+ this.docCache.put(id, doc);
249
+ doc._rev = entry.rev;
250
+ }
251
+ return doc;
252
+ }
253
+ /** Generic Download with Caching and Parsing */
254
+ async fetchFile(fileId, skipCache = false) {
255
+ if (!skipCache) {
256
+ const cached = this.fileCache.get(fileId);
257
+ if (cached) {
258
+ this.log('fetchFile cache hit', fileId);
259
+ return cached;
260
+ }
261
+ }
262
+ this.log('fetchFile downloading', fileId);
263
+ const data = await this.client.getFile(fileId);
264
+ let parsed;
265
+ if (typeof data === 'string') {
266
+ const trimmed = data.trim();
267
+ if (trimmed.startsWith('{')) {
268
+ // Could be JSON or NDJSON
269
+ if (trimmed.includes('\n')) {
270
+ // Definitely NDJSON (multiple lines)
271
+ try {
272
+ const lines = trimmed.split('\n').filter(l => l);
273
+ parsed = lines.map(line => JSON.parse(line));
274
+ }
275
+ catch (e) {
276
+ parsed = data;
277
+ }
278
+ }
279
+ else {
280
+ // Single line. Try regular JSON first.
281
+ try {
282
+ parsed = JSON.parse(trimmed);
283
+ // Optional: if we know it was supposed to be NDJSON?
284
+ // Our change files are always NDJSON.
285
+ // But _meta.json and snapshot-*.json are regular JSON.
286
+ }
287
+ catch (e) {
288
+ parsed = data;
289
+ }
290
+ }
291
+ }
292
+ else {
293
+ parsed = data;
294
+ }
295
+ }
296
+ else {
297
+ parsed = data;
298
+ }
299
+ this.fileCache.put(fileId, parsed);
300
+ return parsed;
301
+ }
302
+ /** Get multiple docs (Atomic-ish) used for _allDocs */
303
+ async getMulti(ids) {
304
+ // Naive parallel fetch
305
+ // Optimization: Group by fileID to reduce requests
306
+ const byFile = {};
307
+ const results = {};
308
+ for (const id of ids) {
309
+ const entry = this.index[id];
310
+ if (!entry || entry.deleted) {
311
+ results[id] = null;
312
+ continue;
313
+ }
314
+ // Check cache
315
+ const cached = this.docCache.get(id);
316
+ if (cached) {
317
+ results[id] = cached;
318
+ continue;
319
+ }
320
+ // Group by file
321
+ if (entry.location.fileId === 'LEGACY_MEMORY') {
322
+ // Handle legacy separately
323
+ const doc = await this.get(id); // fallback
324
+ results[id] = doc;
325
+ }
326
+ else {
327
+ if (!byFile[entry.location.fileId])
328
+ byFile[entry.location.fileId] = [];
329
+ byFile[entry.location.fileId].push(id);
330
+ }
331
+ }
332
+ // Fetch files
333
+ for (const [fileId, docIds] of Object.entries(byFile)) {
334
+ try {
335
+ const content = await this.fetchFile(fileId);
336
+ for (const docId of docIds) {
337
+ let doc = null;
338
+ if (Array.isArray(content)) {
339
+ const match = [...content].reverse().find((c) => c.id === docId);
340
+ doc = match ? match.doc : null;
341
+ }
342
+ else if (content && content.docs) {
343
+ doc = content.docs[docId];
344
+ }
345
+ else if (content && content.id === docId && content.doc) {
346
+ // Single ChangeEntry object
347
+ doc = content.doc;
348
+ }
349
+ if (doc) {
350
+ // Add entry.rev to doc just in case
351
+ if (this.index[docId])
352
+ doc._rev = this.index[docId].rev;
353
+ this.docCache.put(docId, doc);
354
+ results[docId] = doc;
355
+ }
356
+ else {
357
+ results[docId] = null;
358
+ }
359
+ }
360
+ }
361
+ catch (e) {
362
+ console.error(`Failed to fetch file ${fileId} for docs ${docIds}`, e);
363
+ // Return nulls
364
+ docIds.forEach(id => results[id] = null);
365
+ }
366
+ }
367
+ return ids.map(id => results[id]);
368
+ }
369
+ /** Return all keys in Index */
370
+ async getIndexKeys() {
371
+ if (this.loadingPromise)
372
+ await this.loadingPromise;
373
+ return Object.keys(this.index);
374
+ }
375
+ /** Get metadata for a specific ID from Index */
376
+ getIndexEntry(id) {
377
+ return this.index[id];
378
+ }
379
+ /** Single change wrapper */
380
+ async appendChange(change) {
381
+ return this.appendChanges([change]);
382
+ }
383
+ /** Append changes with OCC */
384
+ async appendChanges(changes) {
385
+ const MAX_RETRIES = 5;
386
+ let attempt = 0;
387
+ while (attempt < MAX_RETRIES) {
388
+ try {
389
+ return await this.tryAppendChanges(changes);
390
+ }
391
+ catch (err) {
392
+ if (err.status === 412 || err.status === 409) {
393
+ // Reload and RETRY
394
+ await this.load();
395
+ // Check conflicts against Index (Metadata sufficient)
396
+ this.checkConflicts(changes);
397
+ // Reseq
398
+ let currentSeq = this.meta.seq;
399
+ for (const change of changes) {
400
+ currentSeq++;
401
+ change.seq = currentSeq;
402
+ }
403
+ attempt++;
404
+ await new Promise(r => setTimeout(r, Math.random() * 500 + 100));
405
+ continue;
406
+ }
407
+ throw err;
408
+ }
409
+ }
410
+ throw new Error('Failed to append changes');
411
+ }
412
+ async tryAppendChanges(changes) {
413
+ // 1. Write Log File (Upload Data)
414
+ const fileId = await this.writeChangeFile(changes);
415
+ try {
416
+ // 2. Prepare speculative meta update
417
+ const nextMeta = { ...this.meta };
418
+ nextMeta.changeLogIds = [...nextMeta.changeLogIds, fileId];
419
+ nextMeta.seq = changes[changes.length - 1].seq;
420
+ // 3. Commit Lock
421
+ await this.saveMeta(nextMeta, this.metaEtag);
422
+ // 4. Update Local State
423
+ this.meta = nextMeta;
424
+ for (const change of changes) {
425
+ this.updateIndex(change, fileId);
426
+ if (change.doc) {
427
+ this.docCache.put(change.id, change.doc);
428
+ }
429
+ else if (change.deleted) {
430
+ this.docCache.remove(change.id);
431
+ }
432
+ }
433
+ // Notify local changes feed listeners about our own write
434
+ this.notifyListeners();
435
+ // 5. Compaction Check
436
+ const totalChanges = await this.countTotalChanges();
437
+ if (totalChanges >= this.compactionThreshold ||
438
+ this.currentLogSizeEstimate >= this.compactionSizeThreshold) {
439
+ this.compact().catch(e => console.error('Compaction failed', e));
440
+ }
441
+ }
442
+ catch (err) {
443
+ // Cleanup orphaned log file on metadata update failure
444
+ this.client.deleteFile(fileId).catch(e => this.log('Failed to cleanup orphaned log', fileId, e));
445
+ throw err;
446
+ }
447
+ }
448
+ /** Update Index with a new change */
449
+ updateIndex(change, fileId) {
450
+ this.index[change.id] = {
451
+ rev: change.rev,
452
+ seq: change.seq,
453
+ deleted: !!change.deleted,
454
+ location: { fileId }
455
+ };
456
+ }
457
+ checkConflicts(changes) {
458
+ for (const change of changes) {
459
+ const docId = change.id;
460
+ const newRevNum = parseInt(change.rev.split('-')[0], 10);
461
+ const existing = this.index[docId];
462
+ if (existing) {
463
+ const currentRevNum = parseInt(existing.rev.split('-')[0], 10);
464
+ if (currentRevNum >= newRevNum) {
465
+ const err = new Error('Document update conflict');
466
+ err.status = 409;
467
+ err.name = 'conflict'; // PouchDB expectation
468
+ throw err;
469
+ }
470
+ }
471
+ }
472
+ }
473
+ /** Compact: Create SnapshotIndex + SnapshotData */
474
+ async compact() {
475
+ if (this.isCompacting)
476
+ return;
477
+ this.isCompacting = true;
478
+ try {
479
+ this.log('Starting compaction');
480
+ const snapshotSeq = this.meta.seq;
481
+ const oldLogIds = [...this.meta.changeLogIds];
482
+ const oldIndexId = this.meta.snapshotIndexId;
483
+ // 1. Fetch ALL active documents
484
+ // We need them to build the new large snapshot-data file
485
+ // This is the one time we download everything if not cached.
486
+ // Optimization: We could reuse existing `snapshot-data` chunks and only append new data
487
+ // to a new chunk, but for simplicity: Merge All.
488
+ const allIds = Object.keys(this.index).filter(id => !this.index[id].deleted);
489
+ const allDocs = await this.getMulti(allIds);
490
+ const snapshotData = { docs: {} };
491
+ allIds.forEach((id, i) => {
492
+ if (allDocs[i])
493
+ snapshotData.docs[id] = allDocs[i];
494
+ });
495
+ // 2. Upload Data File
496
+ const dataContent = JSON.stringify(snapshotData);
497
+ const dataRes = await this.client.createFile(`snapshot-data-${Date.now()}.json`, [this.folderId], 'application/json', dataContent);
498
+ const dataFileId = dataRes.id;
499
+ // 3. Create Index pointing to this Data File
500
+ const newIndexEntries = {};
501
+ for (const id of Object.keys(snapshotData.docs)) {
502
+ newIndexEntries[id] = {
503
+ rev: this.index[id].rev,
504
+ seq: this.index[id].seq,
505
+ location: { fileId: dataFileId }
506
+ };
507
+ }
508
+ const snapshotIndex = {
509
+ entries: newIndexEntries,
510
+ seq: snapshotSeq,
511
+ createdAt: Date.now()
512
+ };
513
+ const indexContent = JSON.stringify(snapshotIndex);
514
+ const indexRes = await this.client.createFile(`snapshot-index-${Date.now()}.json`, [this.folderId], 'application/json', indexContent);
515
+ const newIndexId = indexRes.id;
516
+ // 4. Update Meta
517
+ let filesToDelete = [];
518
+ await this.atomicUpdateMeta((latest) => {
519
+ const remainingLogs = latest.changeLogIds.filter(id => !oldLogIds.includes(id));
520
+ // Only delete files that were in oldLogIds but not in remainingLogs
521
+ filesToDelete = oldLogIds.filter(id => !remainingLogs.includes(id));
522
+ return {
523
+ ...latest,
524
+ snapshotIndexId: newIndexId,
525
+ changeLogIds: remainingLogs,
526
+ lastCompaction: Date.now()
527
+ };
528
+ });
529
+ // 5. Cleanup - Only delete files that were confirmed removed from metadata
530
+ await this.cleanupOldFiles(oldIndexId, filesToDelete);
531
+ this.currentLogSizeEstimate = 0;
532
+ }
533
+ finally {
534
+ this.isCompacting = false;
535
+ }
536
+ }
537
+ // ... Helpers (atomicUpdateMeta, saveMeta, writeChangeFile same as before) ...
538
+ async atomicUpdateMeta(modifier) {
539
+ const MAX_RETRIES = 5;
540
+ let attempt = 0;
541
+ while (attempt < MAX_RETRIES) {
542
+ try {
543
+ const metaFile = await this.findFile('_meta.json');
544
+ if (!metaFile)
545
+ throw new Error('Meta missing');
546
+ const validMeta = await this.downloadJson(metaFile.id, true); // No cache
547
+ const newMeta = modifier(validMeta);
548
+ await this.saveMeta(newMeta, metaFile.etag);
549
+ this.meta = newMeta;
550
+ return;
551
+ }
552
+ catch (err) {
553
+ if (err.status === 412 || err.status === 409) {
554
+ attempt++;
555
+ await new Promise(r => setTimeout(r, Math.random() * 500 + 100));
556
+ continue;
557
+ }
558
+ throw err;
559
+ }
560
+ }
561
+ }
562
+ // Reused helpers
563
+ async findOrCreateFolder() {
564
+ const safeName = this.escapeQuery(this.folderName);
565
+ const q = `name = '${safeName}' and mimeType = 'application/vnd.google-apps.folder' and trashed = false`;
566
+ const files = await this.client.listFiles(q);
567
+ if (files.length > 0)
568
+ return files[0].id;
569
+ const createRes = await this.client.createFile(this.folderName, this.parents.length ? this.parents : undefined, 'application/vnd.google-apps.folder', '');
570
+ return createRes.id;
571
+ }
572
+ async findFile(name) {
573
+ if (!this.folderId)
574
+ return null;
575
+ const safeName = this.escapeQuery(name);
576
+ const q = `name = '${safeName}' and '${this.folderId}' in parents and trashed = false`;
577
+ const files = await this.client.listFiles(q);
578
+ if (files.length > 0) {
579
+ let file = files[0];
580
+ if (!file.etag) {
581
+ // Robustness: Fetch metadata for the file if etag is missing from list
582
+ try {
583
+ file = await this.client.getFileMetadata(file.id);
584
+ }
585
+ catch (e) {
586
+ this.log('Failed to fetch file metadata for etag', file.id, e);
587
+ }
588
+ }
589
+ return {
590
+ id: file.id,
591
+ etag: file.etag,
592
+ modifiedTime: file.modifiedTime
593
+ };
594
+ }
595
+ return null;
596
+ }
597
+ async downloadJson(fileId, skipCache = false) {
598
+ return await this.fetchFile(fileId, skipCache);
599
+ }
600
+ async downloadFileAny(fileId) {
601
+ return await this.fetchFile(fileId);
602
+ }
603
+ async downloadNdjson(fileId) {
604
+ return await this.fetchFile(fileId);
605
+ }
606
+ async writeChangeFile(changes) {
607
+ const lines = changes.map(c => JSON.stringify(c)).join('\n') + '\n';
608
+ const startSeq = changes[0].seq;
609
+ const name = `changes-${startSeq}-${Math.random().toString(36).substring(7)}.ndjson`;
610
+ const res = await this.client.createFile(name, [this.folderId], 'application/x-ndjson', lines);
611
+ this.currentLogSizeEstimate += new Blob([lines]).size;
612
+ return res.id;
613
+ }
614
+ async saveMeta(meta, expectedEtag = null) {
615
+ const content = JSON.stringify(meta);
616
+ const metaFile = await this.findFile('_meta.json');
617
+ if (metaFile) {
618
+ const res = await this.client.updateFile(metaFile.id, content, expectedEtag || undefined);
619
+ this.metaEtag = res.etag;
620
+ this.metaModifiedTime = res.modifiedTime;
621
+ this.fileCache.remove(metaFile.id); // Invalidate cache
622
+ }
623
+ else {
624
+ const res = await this.client.createFile('_meta.json', [this.folderId], 'application/json', content);
625
+ this.metaEtag = res.etag;
626
+ this.metaModifiedTime = res.modifiedTime;
627
+ }
628
+ }
629
+ async countTotalChanges() {
630
+ // If no snapshot exists yet, total changes = meta.seq (all changes)
631
+ if (!this.meta.snapshotIndexId) {
632
+ return this.meta.seq;
633
+ }
634
+ // Each log file ID in changeLogIds represents some number of changes.
635
+ // For simplicity and to trigger compaction based on file count (which is what matters for Drive),
636
+ // we can return the number of log files.
637
+ // But since compactionThreshold is usually in ENTRIES, let's keep a rough estimate
638
+ // or just return the log file count if that's what the user expects.
639
+ // The previous "* 5" was too aggressive.
640
+ // Let's assume on average 1 change per log file in tests (worst case).
641
+ return this.meta.changeLogIds.length + this.pendingChanges.length;
642
+ }
643
+ async cleanupOldFiles(oldIndexId, oldLogIds) {
644
+ const deleteFile = async (fileId) => {
645
+ try {
646
+ await this.client.deleteFile(fileId);
647
+ this.log('Deleted file', fileId);
648
+ }
649
+ catch (err) {
650
+ // 404 is ok - file already deleted or doesn't exist
651
+ if (err.status === 404 || err.code === 404) {
652
+ this.log('File already deleted or not found', fileId);
653
+ return;
654
+ }
655
+ // Log other errors but don't fail
656
+ this.log('Failed to delete file', fileId, err);
657
+ }
658
+ };
659
+ if (oldIndexId) {
660
+ await deleteFile(oldIndexId);
661
+ }
662
+ for (const id of oldLogIds) {
663
+ await deleteFile(id);
664
+ }
665
+ }
666
+ startPolling(intervalMs) {
667
+ this.log('Starting polling with interval', { intervalMs });
668
+ if (isNaN(intervalMs) || intervalMs <= 0)
669
+ return;
670
+ if (this.pollingInterval)
671
+ clearInterval(this.pollingInterval);
672
+ this.pollingInterval = setInterval(async () => {
673
+ this.log('Polling tick...');
674
+ if (this.isPollingActive) {
675
+ this.log('Polling already in progress, skipping tick');
676
+ return;
677
+ }
678
+ this.isPollingActive = true;
679
+ try {
680
+ const metaFile = await this.findFile('_meta.json');
681
+ if (!metaFile) {
682
+ this.log('Polling: _meta.json not found');
683
+ return;
684
+ }
685
+ // Compare etags, falling back to modifiedTime
686
+ this.log('Polling: comparing etag', metaFile.etag, 'with', this.metaEtag);
687
+ if ((metaFile.etag && this.metaEtag && metaFile.etag !== this.metaEtag) ||
688
+ (!metaFile.etag && metaFile.modifiedTime !== this.metaModifiedTime)) {
689
+ this.log('Polling detected change!', metaFile.etag || metaFile.modifiedTime);
690
+ await this.load();
691
+ this.notifyListeners();
692
+ }
693
+ }
694
+ catch (err) {
695
+ this.log('Polling error', err);
696
+ }
697
+ finally {
698
+ this.isPollingActive = false;
699
+ }
700
+ }, intervalMs);
701
+ }
702
+ notifyListeners() {
703
+ // Observers expecting 'docs' object might be broken if they expect FULL body.
704
+ // We can pass empty object or partials?
705
+ // Real PouchDB changes feed calls `db.changes()`.
706
+ // Our `adapter.js` uses `db.onChange` effectively.
707
+ // We should pass a map of { ID: { _rev, ... } } (Index entries)
708
+ // Adapter needs to handle this.
709
+ const changes = {};
710
+ for (const [id, entry] of Object.entries(this.index)) {
711
+ changes[id] = {
712
+ _id: id,
713
+ _rev: entry.rev,
714
+ _deleted: !!entry.deleted,
715
+ seq: entry.seq // IMPORTANT: Missing previously, preventing filtered changes from working
716
+ };
717
+ }
718
+ for (const cb of this.listeners)
719
+ cb(changes);
720
+ }
721
+ // For tests/debug
722
+ onChange(cb) {
723
+ this.listeners.push(cb);
724
+ return () => {
725
+ this.listeners = this.listeners.filter(l => l !== cb);
726
+ };
727
+ }
728
+ stopPolling() { if (this.pollingInterval)
729
+ clearInterval(this.pollingInterval); }
730
+ escapeQuery(value) {
731
+ return value.replace(/'/g, "\\'");
732
+ }
733
+ async deleteFolder() { if (this.folderId)
734
+ await this.client.deleteFile(this.folderId); }
735
+ getNextSeq() { return this.meta.seq + 1; }
736
+ }
737
+ exports.DriveHandler = DriveHandler;