@hold-baby/bucket-db 0.2.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. package/README.md +234 -19
  2. package/dist/cache/lru-cache.d.ts +47 -0
  3. package/dist/cache/lru-cache.d.ts.map +1 -0
  4. package/dist/cache/lru-cache.js +110 -0
  5. package/dist/cache/lru-cache.js.map +1 -0
  6. package/dist/core/bucket-db.d.ts +20 -0
  7. package/dist/core/bucket-db.d.ts.map +1 -0
  8. package/dist/core/bucket-db.js +32 -0
  9. package/dist/core/bucket-db.js.map +1 -0
  10. package/dist/core/collection.d.ts +26 -0
  11. package/dist/core/collection.d.ts.map +1 -0
  12. package/dist/core/collection.js +213 -0
  13. package/dist/core/collection.js.map +1 -0
  14. package/dist/core/wal.d.ts +45 -0
  15. package/dist/core/wal.d.ts.map +1 -0
  16. package/dist/core/wal.js +80 -0
  17. package/dist/core/wal.js.map +1 -0
  18. package/dist/index/shard-manager.d.ts +36 -0
  19. package/dist/index/shard-manager.d.ts.map +1 -0
  20. package/dist/index/shard-manager.js +177 -0
  21. package/dist/index/shard-manager.js.map +1 -0
  22. package/dist/index.d.ts +9 -0
  23. package/dist/index.d.ts.map +1 -0
  24. package/dist/index.js +10 -0
  25. package/dist/index.js.map +1 -0
  26. package/dist/query/evaluator.d.ts +10 -0
  27. package/dist/query/evaluator.d.ts.map +1 -0
  28. package/dist/query/evaluator.js +127 -0
  29. package/dist/query/evaluator.js.map +1 -0
  30. package/dist/storage/fs-adapter.d.ts +24 -0
  31. package/dist/storage/fs-adapter.d.ts.map +1 -0
  32. package/dist/storage/fs-adapter.js +208 -0
  33. package/dist/storage/fs-adapter.js.map +1 -0
  34. package/dist/storage/memory-adapter.d.ts +22 -0
  35. package/dist/storage/memory-adapter.d.ts.map +1 -0
  36. package/dist/storage/memory-adapter.js +78 -0
  37. package/dist/storage/memory-adapter.js.map +1 -0
  38. package/dist/storage/oss-adapter.d.ts +26 -0
  39. package/dist/storage/oss-adapter.d.ts.map +1 -0
  40. package/dist/storage/oss-adapter.js +145 -0
  41. package/dist/storage/oss-adapter.js.map +1 -0
  42. package/dist/storage/s3-adapter.d.ts +26 -0
  43. package/dist/storage/s3-adapter.d.ts.map +1 -0
  44. package/dist/storage/s3-adapter.js +170 -0
  45. package/dist/storage/s3-adapter.js.map +1 -0
  46. package/dist/types/collection.d.ts +40 -0
  47. package/dist/types/collection.d.ts.map +1 -0
  48. package/dist/types/collection.js +2 -0
  49. package/dist/types/collection.js.map +1 -0
  50. package/{src/types/document.ts → dist/types/document.d.ts} +6 -10
  51. package/dist/types/document.d.ts.map +1 -0
  52. package/dist/types/document.js +2 -0
  53. package/dist/types/document.js.map +1 -0
  54. package/dist/types/errors.d.ts +32 -0
  55. package/dist/types/errors.d.ts.map +1 -0
  56. package/dist/types/errors.js +48 -0
  57. package/dist/types/errors.js.map +1 -0
  58. package/dist/types/index.d.ts +6 -0
  59. package/dist/types/index.d.ts.map +1 -0
  60. package/dist/types/index.js +3 -0
  61. package/dist/types/index.js.map +1 -0
  62. package/dist/types/query.d.ts +37 -0
  63. package/dist/types/query.d.ts.map +1 -0
  64. package/dist/types/query.js +2 -0
  65. package/dist/types/query.js.map +1 -0
  66. package/dist/types/storage.d.ts +45 -0
  67. package/dist/types/storage.d.ts.map +1 -0
  68. package/dist/types/storage.js +2 -0
  69. package/dist/types/storage.js.map +1 -0
  70. package/dist/utils/hash.d.ts +13 -0
  71. package/dist/utils/hash.d.ts.map +1 -0
  72. package/dist/utils/hash.js +23 -0
  73. package/dist/utils/hash.js.map +1 -0
  74. package/dist/utils/path-validator.d.ts +46 -0
  75. package/dist/utils/path-validator.d.ts.map +1 -0
  76. package/dist/utils/path-validator.js +99 -0
  77. package/dist/utils/path-validator.js.map +1 -0
  78. package/llms.txt +46 -0
  79. package/package.json +14 -3
  80. package/src/core/bucket-db.ts +0 -43
  81. package/src/core/collection.ts +0 -173
  82. package/src/index/shard-manager.ts +0 -142
  83. package/src/index.ts +0 -37
  84. package/src/query/evaluator.ts +0 -87
  85. package/src/storage/ali-oss.d.ts +0 -43
  86. package/src/storage/fs-adapter.ts +0 -222
  87. package/src/storage/memory-adapter.ts +0 -105
  88. package/src/storage/oss-adapter.ts +0 -122
  89. package/src/storage/s3-adapter.ts +0 -151
  90. package/src/types/collection.ts +0 -34
  91. package/src/types/errors.ts +0 -49
  92. package/src/types/index.ts +0 -36
  93. package/src/types/query.ts +0 -50
  94. package/src/types/storage.ts +0 -46
  95. package/src/utils/hash.ts +0 -24
  96. package/tests/core/bucket-db.test.d.ts +0 -2
  97. package/tests/core/bucket-db.test.d.ts.map +0 -1
  98. package/tests/core/bucket-db.test.js +0 -34
  99. package/tests/core/bucket-db.test.js.map +0 -1
  100. package/tests/core/bucket-db.test.ts +0 -50
  101. package/tests/core/collection.test.d.ts +0 -2
  102. package/tests/core/collection.test.d.ts.map +0 -1
  103. package/tests/core/collection.test.js +0 -148
  104. package/tests/core/collection.test.js.map +0 -1
  105. package/tests/core/collection.test.ts +0 -195
  106. package/tests/core/dbpath.test.ts +0 -52
  107. package/tests/index/shard-manager.test.d.ts +0 -2
  108. package/tests/index/shard-manager.test.d.ts.map +0 -1
  109. package/tests/index/shard-manager.test.js +0 -106
  110. package/tests/index/shard-manager.test.js.map +0 -1
  111. package/tests/index/shard-manager.test.ts +0 -129
  112. package/tests/query/evaluator.test.d.ts +0 -2
  113. package/tests/query/evaluator.test.d.ts.map +0 -1
  114. package/tests/query/evaluator.test.js +0 -87
  115. package/tests/query/evaluator.test.js.map +0 -1
  116. package/tests/query/evaluator.test.ts +0 -104
  117. package/tests/storage/fs-adapter.test.ts +0 -111
  118. package/tests/storage/memory-adapter.test.d.ts +0 -2
  119. package/tests/storage/memory-adapter.test.d.ts.map +0 -1
  120. package/tests/storage/memory-adapter.test.js +0 -65
  121. package/tests/storage/memory-adapter.test.js.map +0 -1
  122. package/tests/storage/memory-adapter.test.ts +0 -84
  123. package/tests/storage/oss-adapter.test.d.ts +0 -2
  124. package/tests/storage/oss-adapter.test.d.ts.map +0 -1
  125. package/tests/storage/oss-adapter.test.js +0 -34
  126. package/tests/storage/oss-adapter.test.js.map +0 -1
  127. package/tests/storage/oss-adapter.test.ts +0 -40
  128. package/tests/storage/s3-adapter.test.d.ts +0 -2
  129. package/tests/storage/s3-adapter.test.d.ts.map +0 -1
  130. package/tests/storage/s3-adapter.test.js +0 -51
  131. package/tests/storage/s3-adapter.test.js.map +0 -1
  132. package/tests/storage/s3-adapter.test.ts +0 -62
  133. package/tests/utils/hash.test.d.ts +0 -2
  134. package/tests/utils/hash.test.d.ts.map +0 -1
  135. package/tests/utils/hash.test.js +0 -36
  136. package/tests/utils/hash.test.js.map +0 -1
  137. package/tests/utils/hash.test.ts +0 -42
  138. package/tsconfig.json +0 -12
@@ -0,0 +1,213 @@
1
+ import { DocumentNotFoundError, ConcurrentUpdateError } from '../types/index.js';
2
+ import { ShardManager } from '../index/shard-manager.js';
3
+ import { sanitizePathComponent, safePathJoin } from '../utils/path-validator.js';
4
+ import { WAL } from './wal.js';
5
+ import { LRUCache } from '../cache/lru-cache.js';
6
+ /**
7
+ * Generate UUID v4
8
+ */
9
+ function generateId() {
10
+ return crypto.randomUUID();
11
+ }
12
+ /**
13
+ * Get current ISO timestamp
14
+ */
15
+ function now() {
16
+ return new Date().toISOString();
17
+ }
18
+ /**
19
+ * Collection class for type-safe CRUD operations
20
+ */
21
+ export class Collection {
22
+ adapter;
23
+ shardManager;
24
+ wal;
25
+ queryCache;
26
+ basePath;
27
+ constructor(adapter, dbPath, collectionName, options = {}) {
28
+ this.adapter = adapter;
29
+ // Validate collection name to prevent path traversal
30
+ const sanitizedCollectionName = sanitizePathComponent(collectionName);
31
+ this.basePath = safePathJoin(dbPath, sanitizedCollectionName);
32
+ this.shardManager = new ShardManager(adapter, this.basePath, options.shardCount || 16);
33
+ this.wal = new WAL(adapter, this.basePath);
34
+ this.queryCache = new LRUCache(100, 5 * 60 * 1000); // 100 entries, 5 min TTL
35
+ // Recover from any pending WAL entries
36
+ this.recoverFromWAL().catch(err => {
37
+ console.error('WAL recovery failed:', err);
38
+ });
39
+ }
40
+ /**
41
+ * Recover from pending WAL entries
42
+ */
43
+ async recoverFromWAL() {
44
+ await this.wal.recover(async (entry) => {
45
+ // Replay the operation based on WAL entry
46
+ switch (entry.operation) {
47
+ case 'insert':
48
+ case 'update':
49
+ // Ensure document is indexed
50
+ if (entry.documentData) {
51
+ await this.shardManager.addDocument(entry.documentData);
52
+ }
53
+ break;
54
+ case 'delete':
55
+ // Ensure document is removed from index
56
+ await this.shardManager.removeDocument(entry.documentId);
57
+ break;
58
+ }
59
+ });
60
+ }
61
+ getDocPath(id) {
62
+ // Validate document ID to prevent path traversal
63
+ const sanitizedId = sanitizePathComponent(id);
64
+ return safePathJoin(this.basePath, 'docs', `${sanitizedId}.json`);
65
+ }
66
+ async insert(data) {
67
+ const id = generateId();
68
+ const timestamp = now();
69
+ const document = {
70
+ ...data,
71
+ id,
72
+ _createdAt: timestamp,
73
+ _updatedAt: timestamp,
74
+ };
75
+ // 1. Log operation to WAL first
76
+ await this.wal.log({
77
+ operation: 'insert',
78
+ collectionPath: this.basePath,
79
+ documentId: id,
80
+ documentData: document,
81
+ timestamp,
82
+ });
83
+ try {
84
+ // 2. Write document to storage
85
+ const { etag } = await this.adapter.put(this.getDocPath(id), document);
86
+ document._etag = etag;
87
+ // 3. Add to index
88
+ await this.shardManager.addDocument(document);
89
+ // 4. Clear WAL entry after successful completion
90
+ await this.wal.clear(id);
91
+ // 5. Invalidate query cache (new document may match existing queries)
92
+ this.queryCache.invalidate();
93
+ return document;
94
+ }
95
+ catch (error) {
96
+ // On failure, WAL entry remains for recovery
97
+ throw error;
98
+ }
99
+ }
100
+ async findById(id) {
101
+ try {
102
+ const { data, etag } = await this.adapter.get(this.getDocPath(id));
103
+ const doc = data;
104
+ doc._etag = etag;
105
+ return doc;
106
+ }
107
+ catch (error) {
108
+ return null;
109
+ }
110
+ }
111
+ async find(filter, options) {
112
+ // Check cache first
113
+ const cacheKey = LRUCache.generateKey(filter, options);
114
+ const cached = this.queryCache.get(cacheKey);
115
+ if (cached) {
116
+ return cached;
117
+ }
118
+ // Query index to get matching document IDs
119
+ const indexResults = await this.shardManager.query(filter);
120
+ // Get full documents
121
+ let ids = indexResults.map(doc => doc.id);
122
+ if (ids.length === 0) {
123
+ return [];
124
+ }
125
+ // Apply pagination on IDs before loading documents (PERF optimization)
126
+ if (options?.offset) {
127
+ ids = ids.slice(options.offset);
128
+ }
129
+ if (options?.limit) {
130
+ ids = ids.slice(0, options.limit);
131
+ }
132
+ // Batch get only the required documents
133
+ const docMap = await this.adapter.batchGet(ids.map(id => this.getDocPath(id)));
134
+ // Convert to array and add etags
135
+ const results = Array.from(docMap.values()).map(obj => {
136
+ const doc = obj.data;
137
+ doc._etag = obj.etag;
138
+ return doc;
139
+ });
140
+ // Cache the results
141
+ this.queryCache.set(cacheKey, results);
142
+ return results;
143
+ }
144
+ async update(id, data, options) {
145
+ // Get current document
146
+ const current = await this.findById(id);
147
+ if (!current) {
148
+ throw new DocumentNotFoundError(id);
149
+ }
150
+ // Check optimistic lock if etag provided
151
+ if (options?.etag && current._etag !== options.etag) {
152
+ throw new ConcurrentUpdateError(`ETag mismatch for document ${id}`);
153
+ }
154
+ // Merge updates
155
+ const updated = {
156
+ ...current,
157
+ ...data,
158
+ id,
159
+ _updatedAt: now(),
160
+ };
161
+ // 1. Log operation to WAL first
162
+ await this.wal.log({
163
+ operation: 'update',
164
+ collectionPath: this.basePath,
165
+ documentId: id,
166
+ documentData: updated,
167
+ timestamp: now(),
168
+ });
169
+ try {
170
+ // 2. Write document
171
+ const { etag } = await this.adapter.put(this.getDocPath(id), updated, current._etag ? { ifMatch: current._etag } : undefined);
172
+ updated._etag = etag;
173
+ // 3. Update index
174
+ await this.shardManager.updateDocument(updated);
175
+ // 4. Clear WAL entry
176
+ await this.wal.clear(id);
177
+ // 5. Invalidate query cache (document changed)
178
+ this.queryCache.invalidate();
179
+ return updated;
180
+ }
181
+ catch (error) {
182
+ throw error;
183
+ }
184
+ }
185
+ async delete(id) {
186
+ // Check if document exists
187
+ const exists = await this.findById(id);
188
+ if (!exists) {
189
+ throw new DocumentNotFoundError(id);
190
+ }
191
+ // 1. Log operation to WAL first
192
+ await this.wal.log({
193
+ operation: 'delete',
194
+ collectionPath: this.basePath,
195
+ documentId: id,
196
+ timestamp: now(),
197
+ });
198
+ try {
199
+ // 2. Delete from storage
200
+ await this.adapter.delete(this.getDocPath(id));
201
+ // 3. Remove from index
202
+ await this.shardManager.removeDocument(id);
203
+ // 4. Clear WAL entry
204
+ await this.wal.clear(id);
205
+ // 5. Invalidate query cache (document deleted)
206
+ this.queryCache.invalidate();
207
+ }
208
+ catch (error) {
209
+ throw error;
210
+ }
211
+ }
212
+ }
213
+ //# sourceMappingURL=collection.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"collection.js","sourceRoot":"","sources":["../../src/core/collection.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,qBAAqB,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AACjF,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAAE,qBAAqB,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AACjF,OAAO,EAAE,GAAG,EAAiB,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAEjD;;GAEG;AACH,SAAS,UAAU;IACjB,OAAO,MAAM,CAAC,UAAU,EAAE,CAAC;AAC7B,CAAC;AAED;;GAEG;AACH,SAAS,GAAG;IACV,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AAClC,CAAC;AAMD;;GAEG;AACH,MAAM,OAAO,UAAU;IAOX;IANF,YAAY,CAAe;IAC3B,GAAG,CAAM;IACT,UAAU,CAAgB;IAC1B,QAAQ,CAAS;IAEzB,YACU,OAAuB,EAC/B,MAAc,EACd,cAAsB,EACtB,UAA6B,EAAE;QAHvB,YAAO,GAAP,OAAO,CAAgB;QAK/B,qDAAqD;QACrD,MAAM,uBAAuB,GAAG,qBAAqB,CAAC,cAAc,CAAC,CAAC;QACtE,IAAI,CAAC,QAAQ,GAAG,YAAY,CAAC,MAAM,EAAE,uBAAuB,CAAC,CAAC;QAC9D,IAAI,CAAC,YAAY,GAAG,IAAI,YAAY,CAClC,OAAO,EACP,IAAI,CAAC,QAAQ,EACb,OAAO,CAAC,UAAU,IAAI,EAAE,CACzB,CAAC;QACF,IAAI,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC3C,IAAI,CAAC,UAAU,GAAG,IAAI,QAAQ,CAAM,GAAG,EAAE,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,yBAAyB;QAElF,uCAAuC;QACvC,IAAI,CAAC,cAAc,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE;YAChC,OAAO,CAAC,KAAK,CAAC,sBAAsB,EAAE,GAAG,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc;QAC1B,MAAM,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,KAAe,EAAE,EAAE;YAC/C,0CAA0C;YAC1C,QAAQ,KAAK,CAAC,SAAS,EAAE,CAAC;gBACxB,KAAK,QAAQ,CAAC;gBACd,KAAK,QAAQ;oBACX,6BAA6B;oBAC7B,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;wBACvB,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;oBAC1D,CAAC;oBACD,MAAM;gBACR,KAAK,QAAQ;oBACX,wCAAwC;oBACxC,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;oBACzD,MAAM;YACV,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,UAAU,CAAC,EAAU;QAC3B,iDAAiD;QACjD,MAAM,WAAW,GAAG,qBAAqB,CAAC,EAAE,CAAC,CAAC;QAC9C,OAAO,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,WAAW,OAAO,CAAC,CAAC;IACpE,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAuB;QAClC,MAAM,EAAE,GAAG,UAAU,EAAE,CAAC;QACxB,MAAM,SAAS,GAAG,GAAG,EAAE,CAAC;QAExB,MAAM,QAAQ,GAAM;YAClB,GAAG,IAAI;YACP,EAAE;YACF,UAAU,EAAE,SAAS;YACrB,UAAU,EAAE,SAAS;SACjB,CAAC;QAEP,gCAAgC;QAChC,MAAM,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YACjB,SAAS,EAAE,QAAQ;YACnB,cAAc,EAAE,IAAI,CAAC,QAAQ;YAC7B,UAAU,EAAE,EAAE;YACd,YAAY,EAAE,QAAQ;YACtB,SAAS;SACV,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,+BAA+B;YAC/B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC;YACvE,QAAQ,CAAC,KAAK,GAAG,IAAI,CAAC;YAEtB,kBAAkB;YAClB,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;YAE9C,iDAAiD;YACjD,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAEzB,sEAAsE;YACtE,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC;YAE7B,OAAO,QAAQ,CAAC;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,6CAA6C;YAC7C,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,EAAU;QACvB,IAAI,CAAC;YACH,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC;YACnE,MAAM,GAAG,GAAG,IAAS,CAAC;YACtB,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC;YACjB,OAAO,GAAG,CAAC;QACb,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAsB,EAAE,OAAsB;QACvD,oBAAoB;QACpB,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,CAAC,MAA6B,EAAE,OAA8B,CAAC,CAAC;QACrG,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC7C,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,2CAA2C;QAC3C,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAE3D,qBAAqB;QACrB,IAAI,GAAG,GAAG,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAE1C,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,uEAAuE;QACvE,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;YACpB,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC;QACD,IAAI,OAAO,EAAE,KAAK,EAAE,CAAC;YACnB,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QACpC,CAAC;QAED,wCAAwC;QACxC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAE/E,iCAAiC;QACjC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YACpD,MAAM,GAAG,GAAG,GAAG,CAAC,IAAS,CAAC;YAC1B,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC;YACrB,OAAO,GAAG,CAAC;QACb,CAAC,CAAC,CAAC;QAEH,oBAAoB;QACpB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAEvC,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,MAAM,CACV,EAAU,EACV,IAAuB,EACvB,OAAuB;QAEvB,uBAAuB;QACvB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QACxC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,qBAAqB,CAAC,EAAE,CAAC,CAAC;QACtC,CAAC;QAED,yCAAyC;QACzC,IAAI,OAAO,EAAE,IAAI,IAAI,OAAO,CAAC,KAAK,KAAK,OAAO,CAAC,IAAI,EAAE,CAAC;YACpD,MAAM,IAAI,qBAAqB,CAAC,8BAA8B,EAAE,EAAE,CAAC,CAAC;QACtE,CAAC;QAED,gBAAgB;QAChB,MAAM,OAAO,GAAM;YACjB,GAAG,OAAO;YACV,GAAG,IAAI;YACP,EAAE;YACF,UAAU,EAAE,GAAG,EAAE;SACb,CAAC;QAEP,gCAAgC;QAChC,MAAM,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YACjB,SAAS,EAAE,QAAQ;YACnB,cAAc,EAAE,IAAI,CAAC,QAAQ;YAC7B,UAAU,EAAE,EAAE;YACd,YAAY,EAAE,OAAO;YACrB,SAAS,EAAE,GAAG,EAAE;SACjB,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,oBAAoB;YACpB,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CACrC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,EACnB,OAAO,EACP,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CACvD,CAAC;YAEF,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC;YAErB,kBAAkB;YAClB,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YAEhD,qBAAqB;YACrB,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAEzB,+CAA+C;YAC/C,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC;YAE7B,OAAO,OAAO,CAAC;QACjB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAU;QACrB,2BAA2B;QAC3B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QACvC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,qBAAqB,CAAC,EAAE,CAAC,CAAC;QACtC,CAAC;QAED,gCAAgC;QAChC,MAAM,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YACjB,SAAS,EAAE,QAAQ;YACnB,cAAc,EAAE,IAAI,CAAC,QAAQ;YAC7B,UAAU,EAAE,EAAE;YACd,SAAS,EAAE,GAAG,EAAE;SACjB,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,yBAAyB;YACzB,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC;YAE/C,uBAAuB;YACvB,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;YAE3C,qBAAqB;YACrB,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAEzB,+CAA+C;YAC/C,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;CACF"}
@@ -0,0 +1,45 @@
1
+ import type { StorageAdapter } from '../types/index.js';
2
+ /**
3
+ * WAL operation types
4
+ */
5
+ export type WALOperation = 'insert' | 'update' | 'delete';
6
+ /**
7
+ * WAL entry structure
8
+ */
9
+ export interface WALEntry {
10
+ operation: WALOperation;
11
+ collectionPath: string;
12
+ documentId: string;
13
+ documentData?: any;
14
+ timestamp: string;
15
+ }
16
+ /**
17
+ * Write-Ahead Log for ensuring index-document consistency
18
+ */
19
+ export declare class WAL {
20
+ private adapter;
21
+ private walPath;
22
+ constructor(adapter: StorageAdapter, basePath: string);
23
+ /**
24
+ * Get WAL entry path for a document
25
+ */
26
+ private getWALEntryPath;
27
+ /**
28
+ * Log an operation before execution
29
+ */
30
+ log(entry: WALEntry): Promise<void>;
31
+ /**
32
+ * Clear WAL entry after successful operation
33
+ */
34
+ clear(documentId: string): Promise<void>;
35
+ /**
36
+ * List all pending WAL entries
37
+ */
38
+ listPending(): Promise<WALEntry[]>;
39
+ /**
40
+ * Recover from pending WAL entries
41
+ * This should be called on collection initialization
42
+ */
43
+ recover(onRecover: (entry: WALEntry) => Promise<void>): Promise<number>;
44
+ }
45
+ //# sourceMappingURL=wal.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"wal.d.ts","sourceRoot":"","sources":["../../src/core/wal.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAGxD;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAE1D;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,SAAS,EAAE,YAAY,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,GAAG,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,qBAAa,GAAG;IAIZ,OAAO,CAAC,OAAO;IAHjB,OAAO,CAAC,OAAO,CAAS;gBAGd,OAAO,EAAE,cAAc,EAC/B,QAAQ,EAAE,MAAM;IAKlB;;OAEG;IACH,OAAO,CAAC,eAAe;IAIvB;;OAEG;IACG,GAAG,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAKzC;;OAEG;IACG,KAAK,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAS9C;;OAEG;IACG,WAAW,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;IAoBxC;;;OAGG;IACG,OAAO,CACX,SAAS,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,OAAO,CAAC,IAAI,CAAC,GAC5C,OAAO,CAAC,MAAM,CAAC;CAiBnB"}
@@ -0,0 +1,80 @@
1
+ import { safePathJoin } from '../utils/path-validator.js';
2
+ /**
3
+ * Write-Ahead Log for ensuring index-document consistency
4
+ */
5
+ export class WAL {
6
+ adapter;
7
+ walPath;
8
+ constructor(adapter, basePath) {
9
+ this.adapter = adapter;
10
+ this.walPath = safePathJoin(basePath, 'wal');
11
+ }
12
+ /**
13
+ * Get WAL entry path for a document
14
+ */
15
+ getWALEntryPath(documentId) {
16
+ return `${this.walPath}/${documentId}.wal.json`;
17
+ }
18
+ /**
19
+ * Log an operation before execution
20
+ */
21
+ async log(entry) {
22
+ const path = this.getWALEntryPath(entry.documentId);
23
+ await this.adapter.put(path, entry);
24
+ }
25
+ /**
26
+ * Clear WAL entry after successful operation
27
+ */
28
+ async clear(documentId) {
29
+ const path = this.getWALEntryPath(documentId);
30
+ try {
31
+ await this.adapter.delete(path);
32
+ }
33
+ catch {
34
+ // Ignore if entry doesn't exist
35
+ }
36
+ }
37
+ /**
38
+ * List all pending WAL entries
39
+ */
40
+ async listPending() {
41
+ try {
42
+ const keys = await this.adapter.listKeys(this.walPath);
43
+ const entries = [];
44
+ for (const key of keys) {
45
+ try {
46
+ const { data } = await this.adapter.get(key);
47
+ entries.push(data);
48
+ }
49
+ catch {
50
+ // Skip corrupted entries
51
+ }
52
+ }
53
+ return entries;
54
+ }
55
+ catch {
56
+ return [];
57
+ }
58
+ }
59
+ /**
60
+ * Recover from pending WAL entries
61
+ * This should be called on collection initialization
62
+ */
63
+ async recover(onRecover) {
64
+ const pending = await this.listPending();
65
+ let recovered = 0;
66
+ for (const entry of pending) {
67
+ try {
68
+ await onRecover(entry);
69
+ await this.clear(entry.documentId);
70
+ recovered++;
71
+ }
72
+ catch (error) {
73
+ // Log error but continue recovery
74
+ console.error(`Failed to recover WAL entry for ${entry.documentId}:`, error);
75
+ }
76
+ }
77
+ return recovered;
78
+ }
79
+ }
80
+ //# sourceMappingURL=wal.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"wal.js","sourceRoot":"","sources":["../../src/core/wal.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAkB1D;;GAEG;AACH,MAAM,OAAO,GAAG;IAIJ;IAHF,OAAO,CAAS;IAExB,YACU,OAAuB,EAC/B,QAAgB;QADR,YAAO,GAAP,OAAO,CAAgB;QAG/B,IAAI,CAAC,OAAO,GAAG,YAAY,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IAC/C,CAAC;IAED;;OAEG;IACK,eAAe,CAAC,UAAkB;QACxC,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,UAAU,WAAW,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG,CAAC,KAAe;QACvB,MAAM,IAAI,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACpD,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK,CAAC,UAAkB;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;QAC9C,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAClC,CAAC;QAAC,MAAM,CAAC;YACP,gCAAgC;QAClC,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW;QACf,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvD,MAAM,OAAO,GAAe,EAAE,CAAC;YAE/B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,IAAI,CAAC;oBACH,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBAC7C,OAAO,CAAC,IAAI,CAAC,IAAgB,CAAC,CAAC;gBACjC,CAAC;gBAAC,MAAM,CAAC;oBACP,yBAAyB;gBAC3B,CAAC;YACH,CAAC;YAED,OAAO,OAAO,CAAC;QACjB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,OAAO,CACX,SAA6C;QAE7C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzC,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,CAAC;gBACH,MAAM,SAAS,CAAC,KAAK,CAAC,CAAC;gBACvB,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBACnC,SAAS,EAAE,CAAC;YACd,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,kCAAkC;gBAClC,OAAO,CAAC,KAAK,CAAC,mCAAmC,KAAK,CAAC,UAAU,GAAG,EAAE,KAAK,CAAC,CAAC;YAC/E,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;CACF"}
@@ -0,0 +1,36 @@
1
+ import type { StorageAdapter, QueryFilter } from '../types/index.js';
2
+ /**
3
+ * Manages index shards for a collection
4
+ */
5
+ export declare class ShardManager {
6
+ private adapter;
7
+ private collectionPath;
8
+ private shardCount;
9
+ constructor(adapter: StorageAdapter, collectionPath: string, shardCount: number);
10
+ private getShardPath;
11
+ private getShardMetadataPath;
12
+ private readShardMetadata;
13
+ private readShard;
14
+ private writeShard;
15
+ /**
16
+ * Add document to index
17
+ */
18
+ addDocument(doc: Record<string, any>): Promise<void>;
19
+ /**
20
+ * Update document in index
21
+ */
22
+ updateDocument(doc: Record<string, any>): Promise<void>;
23
+ /**
24
+ * Remove document from index
25
+ */
26
+ removeDocument(docId: string): Promise<void>;
27
+ /**
28
+ * Find document by ID
29
+ */
30
+ findById(docId: string): Promise<Record<string, any> | null>;
31
+ /**
32
+ * Query documents matching filter
33
+ */
34
+ query(filter: QueryFilter<any>): Promise<Array<Record<string, any>>>;
35
+ }
36
+ //# sourceMappingURL=shard-manager.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"shard-manager.d.ts","sourceRoot":"","sources":["../../src/index/shard-manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAc,WAAW,EAAiB,MAAM,mBAAmB,CAAC;AAKhG;;GAEG;AACH,qBAAa,YAAY;IAErB,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,cAAc;IACtB,OAAO,CAAC,UAAU;gBAFV,OAAO,EAAE,cAAc,EACvB,cAAc,EAAE,MAAM,EACtB,UAAU,EAAE,MAAM;IAG5B,OAAO,CAAC,YAAY;IAIpB,OAAO,CAAC,oBAAoB;YAId,iBAAiB;YAUjB,SAAS;YAiBT,UAAU;IAgBxB;;OAEG;IACG,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA+B1D;;OAEG;IACG,cAAc,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAI7D;;OAEG;IACG,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAiClD;;OAEG;IACG,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI,CAAC;IAMlE;;OAEG;IACG,KAAK,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;CAyC3E"}
@@ -0,0 +1,177 @@
1
+ import { StorageError } from '../types/index.js';
2
+ import { getShardId, formatShardId } from '../utils/hash.js';
3
+ import { evaluateFilter } from '../query/evaluator.js';
4
+ /**
5
+ * Manages index shards for a collection
6
+ */
7
+ export class ShardManager {
8
+ adapter;
9
+ collectionPath;
10
+ shardCount;
11
+ constructor(adapter, collectionPath, shardCount) {
12
+ this.adapter = adapter;
13
+ this.collectionPath = collectionPath;
14
+ this.shardCount = shardCount;
15
+ }
16
+ getShardPath(shardId) {
17
+ return `${this.collectionPath}/index/shard-${formatShardId(shardId)}.json`;
18
+ }
19
+ getShardMetadataPath(shardId) {
20
+ return `${this.collectionPath}/index/shard-${formatShardId(shardId)}.meta.json`;
21
+ }
22
+ async readShardMetadata(shardId) {
23
+ const path = this.getShardMetadataPath(shardId);
24
+ try {
25
+ const { data } = await this.adapter.get(path);
26
+ return data;
27
+ }
28
+ catch (error) {
29
+ return null;
30
+ }
31
+ }
32
+ async readShard(shardId) {
33
+ const path = this.getShardPath(shardId);
34
+ try {
35
+ const { data, etag } = await this.adapter.get(path);
36
+ return { shard: data, etag };
37
+ }
38
+ catch (error) {
39
+ // Return empty shard if not found
40
+ return {
41
+ shard: {
42
+ shardId: formatShardId(shardId),
43
+ documents: {},
44
+ },
45
+ etag: undefined,
46
+ };
47
+ }
48
+ }
49
+ async writeShard(shardId, shard, etag) {
50
+ const path = this.getShardPath(shardId);
51
+ const metaPath = this.getShardMetadataPath(shardId);
52
+ // Write shard data
53
+ await this.adapter.put(path, shard, etag ? { ifMatch: etag } : undefined);
54
+ // Write metadata
55
+ const metadata = {
56
+ shardId: formatShardId(shardId),
57
+ docCount: Object.keys(shard.documents).length,
58
+ lastUpdated: new Date().toISOString(),
59
+ };
60
+ await this.adapter.put(metaPath, metadata);
61
+ }
62
+ /**
63
+ * Add document to index
64
+ */
65
+ async addDocument(doc) {
66
+ const docId = doc.id;
67
+ const shardId = getShardId(docId, this.shardCount);
68
+ // Retry logic for optimistic locking with exponential backoff
69
+ const maxRetries = 3;
70
+ for (let attempt = 0; attempt < maxRetries; attempt++) {
71
+ try {
72
+ // Read shard and etag in a single operation
73
+ const { shard, etag } = await this.readShard(shardId);
74
+ shard.documents[docId] = { ...doc };
75
+ await this.writeShard(shardId, shard, etag);
76
+ return;
77
+ }
78
+ catch (error) {
79
+ if (error instanceof StorageError && error.message.includes('PreconditionFailed')) {
80
+ if (attempt < maxRetries - 1) {
81
+ // Exponential backoff with jitter: 100ms * 2^attempt + random(0-50ms)
82
+ const baseDelay = 100 * Math.pow(2, attempt);
83
+ const jitter = Math.random() * 50;
84
+ await new Promise(resolve => setTimeout(resolve, baseDelay + jitter));
85
+ continue;
86
+ }
87
+ }
88
+ throw error;
89
+ }
90
+ }
91
+ throw new StorageError('Failed to add document after retries');
92
+ }
93
+ /**
94
+ * Update document in index
95
+ */
96
+ async updateDocument(doc) {
97
+ await this.addDocument(doc); // Same as add - upsert behavior
98
+ }
99
+ /**
100
+ * Remove document from index
101
+ */
102
+ async removeDocument(docId) {
103
+ const shardId = getShardId(docId, this.shardCount);
104
+ const maxRetries = 3;
105
+ for (let attempt = 0; attempt < maxRetries; attempt++) {
106
+ try {
107
+ // Read shard and etag in a single operation
108
+ const { shard, etag } = await this.readShard(shardId);
109
+ delete shard.documents[docId];
110
+ await this.writeShard(shardId, shard, etag);
111
+ return;
112
+ }
113
+ catch (error) {
114
+ if (error instanceof StorageError && error.message.includes('PreconditionFailed')) {
115
+ if (attempt < maxRetries - 1) {
116
+ // Exponential backoff with jitter
117
+ const baseDelay = 100 * Math.pow(2, attempt);
118
+ const jitter = Math.random() * 50;
119
+ await new Promise(resolve => setTimeout(resolve, baseDelay + jitter));
120
+ continue;
121
+ }
122
+ }
123
+ // Ignore if shard doesn't exist
124
+ if (error instanceof StorageError && error.message.includes('not found')) {
125
+ return;
126
+ }
127
+ throw error;
128
+ }
129
+ }
130
+ throw new StorageError('Failed to remove document after retries');
131
+ }
132
+ /**
133
+ * Find document by ID
134
+ */
135
+ async findById(docId) {
136
+ const shardId = getShardId(docId, this.shardCount);
137
+ const { shard } = await this.readShard(shardId);
138
+ return shard.documents[docId] || null;
139
+ }
140
+ /**
141
+ * Query documents matching filter
142
+ */
143
+ async query(filter) {
144
+ // Optimization: If filter contains id equality, target specific shard
145
+ if (filter.id && typeof filter.id === 'string') {
146
+ const targetShardId = getShardId(filter.id, this.shardCount);
147
+ const { shard } = await this.readShard(targetShardId);
148
+ const doc = shard.documents[filter.id];
149
+ if (doc && evaluateFilter(doc, filter)) {
150
+ return [doc];
151
+ }
152
+ return [];
153
+ }
154
+ // Read all shard metadata first (lightweight)
155
+ const metadataPromises = Array.from({ length: this.shardCount }, (_, i) => this.readShardMetadata(i));
156
+ const allMetadata = await Promise.all(metadataPromises);
157
+ // Filter out empty shards
158
+ const nonEmptyShardIds = allMetadata
159
+ .map((meta, index) => ({ meta, index }))
160
+ .filter(({ meta }) => meta && meta.docCount > 0)
161
+ .map(({ index }) => index);
162
+ // Read only non-empty shards in parallel
163
+ const shardPromises = nonEmptyShardIds.map(i => this.readShard(i));
164
+ const shardResults = await Promise.all(shardPromises);
165
+ // Filter documents across all shards
166
+ const results = [];
167
+ for (const { shard } of shardResults) {
168
+ for (const doc of Object.values(shard.documents)) {
169
+ if (evaluateFilter(doc, filter)) {
170
+ results.push(doc);
171
+ }
172
+ }
173
+ }
174
+ return results;
175
+ }
176
+ }
177
+ //# sourceMappingURL=shard-manager.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"shard-manager.js","sourceRoot":"","sources":["../../src/index/shard-manager.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD;;GAEG;AACH,MAAM,OAAO,YAAY;IAEb;IACA;IACA;IAHV,YACU,OAAuB,EACvB,cAAsB,EACtB,UAAkB;QAFlB,YAAO,GAAP,OAAO,CAAgB;QACvB,mBAAc,GAAd,cAAc,CAAQ;QACtB,eAAU,GAAV,UAAU,CAAQ;IACzB,CAAC;IAEI,YAAY,CAAC,OAAe;QAClC,OAAO,GAAG,IAAI,CAAC,cAAc,gBAAgB,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC;IAC7E,CAAC;IAEO,oBAAoB,CAAC,OAAe;QAC1C,OAAO,GAAG,IAAI,CAAC,cAAc,gBAAgB,aAAa,CAAC,OAAO,CAAC,YAAY,CAAC;IAClF,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,OAAe;QAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC;QAChD,IAAI,CAAC;YACH,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC9C,OAAO,IAAqB,CAAC;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,OAAe;QACrC,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QACxC,IAAI,CAAC;YACH,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACpD,OAAO,EAAE,KAAK,EAAE,IAAkB,EAAE,IAAI,EAAE,CAAC;QAC7C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,kCAAkC;YAClC,OAAO;gBACL,KAAK,EAAE;oBACL,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC;oBAC/B,SAAS,EAAE,EAAE;iBACd;gBACD,IAAI,EAAE,SAAS;aAChB,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,OAAe,EAAE,KAAiB,EAAE,IAAa;QACxE,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC;QAEpD,mBAAmB;QACnB,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAE1E,iBAAiB;QACjB,MAAM,QAAQ,GAAkB;YAC9B,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC;YAC/B,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,MAAM;YAC7C,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACtC,CAAC;QACF,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,GAAwB;QACxC,MAAM,KAAK,GAAG,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,OAAO,GAAG,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAEnD,8DAA8D;QAC9D,MAAM,UAAU,GAAG,CAAC,CAAC;QACrB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;YACtD,IAAI,CAAC;gBACH,4CAA4C;gBAC5C,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;gBAEtD,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,GAAG,EAAE,CAAC;gBAEpC,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;gBAC5C,OAAO;YACT,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC;oBAClF,IAAI,OAAO,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC;wBAC7B,sEAAsE;wBACtE,MAAM,SAAS,GAAG,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;wBAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC;wBAClC,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC;wBACtE,SAAS;oBACX,CAAC;gBACH,CAAC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;QACD,MAAM,IAAI,YAAY,CAAC,sCAAsC,CAAC,CAAC;IACjE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,GAAwB;QAC3C,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,gCAAgC;IAC/D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,KAAa;QAChC,MAAM,OAAO,GAAG,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAEnD,MAAM,UAAU,GAAG,CAAC,CAAC;QACrB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;YACtD,IAAI,CAAC;gBACH,4CAA4C;gBAC5C,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;gBAEtD,OAAO,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;gBAE9B,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;gBAC5C,OAAO;YACT,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC;oBAClF,IAAI,OAAO,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC;wBAC7B,kCAAkC;wBAClC,MAAM,SAAS,GAAG,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;wBAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC;wBAClC,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC;wBACtE,SAAS;oBACX,CAAC;gBACH,CAAC;gBACD,gCAAgC;gBAChC,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;oBACzE,OAAO;gBACT,CAAC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;QACD,MAAM,IAAI,YAAY,CAAC,yCAAyC,CAAC,CAAC;IACpE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,KAAa;QAC1B,MAAM,OAAO,GAAG,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QACnD,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAChD,OAAO,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC;IACxC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK,CAAC,MAAwB;QAClC,sEAAsE;QACtE,IAAI,MAAM,CAAC,EAAE,IAAI,OAAO,MAAM,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;YAC/C,MAAM,aAAa,GAAG,UAAU,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YAC7D,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;YACtD,MAAM,GAAG,GAAG,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACvC,IAAI,GAAG,IAAI,cAAc,CAAC,GAAG,EAAE,MAAM,CAAC,EAAE,CAAC;gBACvC,OAAO,CAAC,GAAG,CAAC,CAAC;YACf,CAAC;YACD,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,8CAA8C;QAC9C,MAAM,gBAAgB,GAAG,KAAK,CAAC,IAAI,CACjC,EAAE,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,EAC3B,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,CACpC,CAAC;QACF,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QAExD,0BAA0B;QAC1B,MAAM,gBAAgB,GAAG,WAAW;aACjC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;aACvC,MAAM,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;aAC/C,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;QAE7B,yCAAyC;QACzC,MAAM,aAAa,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QACnE,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QAEtD,qCAAqC;QACrC,MAAM,OAAO,GAA+B,EAAE,CAAC;QAC/C,KAAK,MAAM,EAAE,KAAK,EAAE,IAAI,YAAY,EAAE,CAAC;YACrC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;gBACjD,IAAI,cAAc,CAAC,GAAG,EAAE,MAAM,CAAC,EAAE,CAAC;oBAChC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACpB,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}
@@ -0,0 +1,9 @@
1
+ export { BucketDB } from './core/bucket-db.js';
2
+ export { Collection } from './core/collection.js';
3
+ export { MemoryStorageAdapter } from './storage/memory-adapter.js';
4
+ export { S3Adapter } from './storage/s3-adapter.js';
5
+ export { OSSAdapter } from './storage/oss-adapter.js';
6
+ export { FileSystemAdapter } from './storage/fs-adapter.js';
7
+ export type { Document, InsertDocument, UpdateDocument, QueryOperator, QueryValue, QueryFilter, QueryOptions, UpdateOptions, StorageAdapter, StorageObject, PutOptions, StorageCredentials, StorageAdapterConfig, ICollection, CollectionMeta, IndexShard, } from './types/index.js';
8
+ export { BucketDBError, DocumentNotFoundError, ConcurrentUpdateError, ValidationError, StorageError, } from './types/index.js';
9
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAGlD,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAG5D,YAAY,EACV,QAAQ,EACR,cAAc,EACd,cAAc,EACd,aAAa,EACb,UAAU,EACV,WAAW,EACX,YAAY,EACZ,aAAa,EACb,cAAc,EACd,aAAa,EACb,UAAU,EACV,kBAAkB,EAClB,oBAAoB,EACpB,WAAW,EACX,cAAc,EACd,UAAU,GACX,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EACL,aAAa,EACb,qBAAqB,EACrB,qBAAqB,EACrB,eAAe,EACf,YAAY,GACb,MAAM,kBAAkB,CAAC"}
package/dist/index.js ADDED
@@ -0,0 +1,10 @@
1
+ // Core classes
2
+ export { BucketDB } from './core/bucket-db.js';
3
+ export { Collection } from './core/collection.js';
4
+ // Storage adapters
5
+ export { MemoryStorageAdapter } from './storage/memory-adapter.js';
6
+ export { S3Adapter } from './storage/s3-adapter.js';
7
+ export { OSSAdapter } from './storage/oss-adapter.js';
8
+ export { FileSystemAdapter } from './storage/fs-adapter.js';
9
+ export { BucketDBError, DocumentNotFoundError, ConcurrentUpdateError, ValidationError, StorageError, } from './types/index.js';
10
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,eAAe;AACf,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAElD,mBAAmB;AACnB,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAsB5D,OAAO,EACL,aAAa,EACb,qBAAqB,EACrB,qBAAqB,EACrB,eAAe,EACf,YAAY,GACb,MAAM,kBAAkB,CAAC"}
@@ -0,0 +1,10 @@
1
+ import type { QueryValue, QueryFilter } from '../types/index.js';
2
+ /**
3
+ * Evaluates a single operator condition
4
+ */
5
+ export declare function evaluateOperator<T>(value: T, condition: QueryValue<T>): boolean;
6
+ /**
7
+ * Evaluates a complete filter against a document
8
+ */
9
+ export declare function evaluateFilter<T extends Record<string, any>>(document: T, filter: QueryFilter<T>): boolean;
10
+ //# sourceMappingURL=evaluator.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../../src/query/evaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AA4CjE;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAChC,KAAK,EAAE,CAAC,EACR,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC,GACvB,OAAO,CA4DT;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC1D,QAAQ,EAAE,CAAC,EACX,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC,GACrB,OAAO,CAwBT"}