gh-as-db 1.1.0 โ 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +62 -0
- package/dist/core/types.d.ts +16 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/infrastructure/github-storage.d.ts +8 -1
- package/dist/infrastructure/github-storage.js +123 -2
- package/dist/infrastructure/transaction-storage.d.ts +25 -0
- package/dist/infrastructure/transaction-storage.js +54 -0
- package/dist/ui/collection.d.ts +11 -3
- package/dist/ui/collection.js +131 -19
- package/dist/ui/github-db.d.ts +4 -1
- package/dist/ui/github-db.js +7 -1
- package/dist/ui/transaction.d.ts +23 -0
- package/dist/ui/transaction.js +31 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -11,7 +11,11 @@ Use a private GitHub repository as a database for your application. `gh-as-db` p
|
|
|
11
11
|
- ๐ **Secure**: Designed for private repositories using Personal Access Tokens (PAT).
|
|
12
12
|
- ๐ **Performance**: Built-in in-memory caching and auto-indexing for fast local queries.
|
|
13
13
|
- ๐ก๏ธ **Concurrency**: Optimistic locking using Git SHAs to prevent data loss.
|
|
14
|
+
- ๐ **Transactions**: Group multiple operations into a single atomic Git commit.
|
|
15
|
+
- ๐ **Sharding**: One-file-per-document storage strategy for massive collections.
|
|
14
16
|
- ๐งฉ **Middleware**: Extensible hooks for data validation or transformation.
|
|
17
|
+
- ๐ **Validation**: Pluggable schema validation support (Zod, etc.).
|
|
18
|
+
- ๐ **Edge Ready**: Fully compatible with Vercel Edge and Cloudflare Workers.
|
|
15
19
|
- โจ๏ธ **CLI Tool**: Initialize and manage your "database" from the terminal.
|
|
16
20
|
- ๐ฆ **TypeScript**: Fully typed for a great developer experience.
|
|
17
21
|
|
|
@@ -58,6 +62,15 @@ const results = await users.find({
|
|
|
58
62
|
{ field: 'name', operator: 'eq', value: 'John Doe' }
|
|
59
63
|
]
|
|
60
64
|
});
|
|
65
|
+
|
|
66
|
+
// Use Transactions for atomic multi-collection updates
|
|
67
|
+
await db.transaction(async (tx) => {
|
|
68
|
+
const usersTx = tx.collection('users');
|
|
69
|
+
const logsTx = tx.collection('logs');
|
|
70
|
+
|
|
71
|
+
await usersTx.update('1', { name: 'Updated Name' });
|
|
72
|
+
await logsTx.create({ id: 'log-1', action: 'Update user' });
|
|
73
|
+
}, 'Atomic update of user and logs');
|
|
61
74
|
```
|
|
62
75
|
|
|
63
76
|
## API Reference
|
|
@@ -131,6 +144,55 @@ const users = db.collection<User>('users', {
|
|
|
131
144
|
});
|
|
132
145
|
```
|
|
133
146
|
|
|
147
|
+
### Schema Validation
|
|
148
|
+
|
|
149
|
+
You can use any validation library (Zod, Valibot, etc.) by providing a `validator` object.
|
|
150
|
+
|
|
151
|
+
```typescript
|
|
152
|
+
import { z } from 'zod';
|
|
153
|
+
|
|
154
|
+
const userSchema = z.object({
|
|
155
|
+
id: z.string(),
|
|
156
|
+
name: z.string().min(3),
|
|
157
|
+
email: z.string().email(),
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
const users = db.collection<User>('users', {
|
|
161
|
+
validator: {
|
|
162
|
+
validate: (data) => userSchema.parseAsync(data),
|
|
163
|
+
}
|
|
164
|
+
});
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
### Transactions & Batching
|
|
168
|
+
|
|
169
|
+
Transactions allow you to group multiple operations across different collections into a **single Git commit**. This reduces API calls and ensures that either all operations succeed or none are committed.
|
|
170
|
+
|
|
171
|
+
```typescript
|
|
172
|
+
const commitSha = await db.transaction(async (tx) => {
|
|
173
|
+
const posts = tx.collection('posts');
|
|
174
|
+
const count = tx.collection('stats');
|
|
175
|
+
|
|
176
|
+
await posts.create({ id: 'p1', title: 'New Post' });
|
|
177
|
+
await count.update('total', { value: 101 });
|
|
178
|
+
}, 'Create post and increment counter');
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
### Storage Strategies (Sharding)
|
|
182
|
+
|
|
183
|
+
By default, `gh-as-db` stores the entire collection in a single JSON file (`name.json`). For large collections, you can use the `sharded` strategy, which stores **one file per document** (`name/id.json`).
|
|
184
|
+
|
|
185
|
+
```typescript
|
|
186
|
+
const users = db.collection<User>('users', {
|
|
187
|
+
strategy: 'sharded'
|
|
188
|
+
});
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
**Why use Sharding?**
|
|
192
|
+
- ๐ **Performance**: `findById` reads only the specific file, which is much faster than loading a massive JSON array.
|
|
193
|
+
- ๐ **Scalability**: Avoids GitHub's file size limits and reduces merge conflicts.
|
|
194
|
+
- ๐งน **Cleanliness**: Better organization for repositories with thousands of documents.
|
|
195
|
+
|
|
134
196
|
## CLI Usage
|
|
135
197
|
|
|
136
198
|
`gh-as-db` comes with a CLI tool to help you manage your repository.
|
package/dist/core/types.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ export interface GitHubDBConfig {
|
|
|
2
2
|
accessToken: string;
|
|
3
3
|
owner: string;
|
|
4
4
|
repo: string;
|
|
5
|
+
branch?: string;
|
|
5
6
|
cacheTTL?: number;
|
|
6
7
|
}
|
|
7
8
|
export type Schema = Record<string, any>;
|
|
@@ -42,9 +43,24 @@ export interface StorageResponse<T> {
|
|
|
42
43
|
data: T;
|
|
43
44
|
sha: string;
|
|
44
45
|
}
|
|
46
|
+
export interface Validator<T> {
|
|
47
|
+
validate: (data: unknown) => Promise<T> | T;
|
|
48
|
+
}
|
|
49
|
+
export interface CommitChange {
|
|
50
|
+
path: string;
|
|
51
|
+
content: any;
|
|
52
|
+
}
|
|
53
|
+
export type StorageStrategy = "single-file" | "sharded";
|
|
45
54
|
export interface IStorageProvider {
|
|
46
55
|
testConnection(): Promise<boolean>;
|
|
47
56
|
exists(path: string): Promise<boolean>;
|
|
48
57
|
readJson<T>(path: string): Promise<StorageResponse<T>>;
|
|
49
58
|
writeJson<T>(path: string, content: T, message: string, sha?: string): Promise<string>;
|
|
59
|
+
deleteFile(path: string, message: string, sha: string): Promise<void>;
|
|
60
|
+
listDirectory(path: string): Promise<{
|
|
61
|
+
path: string;
|
|
62
|
+
sha: string;
|
|
63
|
+
type: "file" | "dir";
|
|
64
|
+
}[]>;
|
|
65
|
+
commit(changes: CommitChange[], message: string): Promise<string>;
|
|
50
66
|
}
|
package/dist/index.d.ts
CHANGED
package/dist/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { GitHubDBConfig, IStorageProvider, StorageResponse } from "../core/types.js";
|
|
1
|
+
import { GitHubDBConfig, IStorageProvider, StorageResponse, CommitChange } from "../core/types.js";
|
|
2
2
|
import { ICacheProvider } from "./cache-provider.js";
|
|
3
3
|
export declare class GitHubStorageProvider implements IStorageProvider {
|
|
4
4
|
private config;
|
|
@@ -11,4 +11,11 @@ export declare class GitHubStorageProvider implements IStorageProvider {
|
|
|
11
11
|
exists(path: string): Promise<boolean>;
|
|
12
12
|
readJson<T>(path: string): Promise<StorageResponse<T>>;
|
|
13
13
|
writeJson<T>(path: string, content: T, message: string, sha?: string): Promise<string>;
|
|
14
|
+
commit(changes: CommitChange[], message: string): Promise<string>;
|
|
15
|
+
deleteFile(path: string, message: string, sha: string): Promise<void>;
|
|
16
|
+
listDirectory(path: string): Promise<{
|
|
17
|
+
path: string;
|
|
18
|
+
sha: string;
|
|
19
|
+
type: "file" | "dir";
|
|
20
|
+
}[]>;
|
|
14
21
|
}
|
|
@@ -66,7 +66,13 @@ export class GitHubStorageProvider {
|
|
|
66
66
|
if (!("content" in response.data) || !("sha" in response.data)) {
|
|
67
67
|
throw new Error("No content or SHA in response");
|
|
68
68
|
}
|
|
69
|
-
const
|
|
69
|
+
const binary = atob(response.data.content.replace(/\s/g, ""));
|
|
70
|
+
const bytes = new Uint8Array(binary.length);
|
|
71
|
+
for (let i = 0; i < binary.length; i++) {
|
|
72
|
+
bytes[i] = binary.charCodeAt(i);
|
|
73
|
+
}
|
|
74
|
+
const decoder = new TextDecoder("utf-8");
|
|
75
|
+
const content = decoder.decode(bytes);
|
|
70
76
|
const result = {
|
|
71
77
|
data: JSON.parse(content),
|
|
72
78
|
sha: response.data.sha,
|
|
@@ -118,7 +124,9 @@ export class GitHubStorageProvider {
|
|
|
118
124
|
repo: this.config.repo,
|
|
119
125
|
path,
|
|
120
126
|
message,
|
|
121
|
-
content:
|
|
127
|
+
content: btoa(Array.from(new TextEncoder().encode(JSON.stringify(content, null, 2)))
|
|
128
|
+
.map((b) => String.fromCharCode(b))
|
|
129
|
+
.join("")),
|
|
122
130
|
sha: internalSha,
|
|
123
131
|
});
|
|
124
132
|
const newSha = response.data.content?.sha || "";
|
|
@@ -135,4 +143,117 @@ export class GitHubStorageProvider {
|
|
|
135
143
|
throw error;
|
|
136
144
|
}
|
|
137
145
|
}
|
|
146
|
+
async commit(changes, message) {
|
|
147
|
+
if (changes.length === 0) {
|
|
148
|
+
throw new Error("No changes to commit");
|
|
149
|
+
}
|
|
150
|
+
const branch = this.config.branch || "main";
|
|
151
|
+
try {
|
|
152
|
+
// 1. Get the current head SHA
|
|
153
|
+
const { data: ref } = await this.octokit.git.getRef({
|
|
154
|
+
owner: this.config.owner,
|
|
155
|
+
repo: this.config.repo,
|
|
156
|
+
ref: `heads/${branch}`,
|
|
157
|
+
});
|
|
158
|
+
const latestCommitSha = ref.object.sha;
|
|
159
|
+
// 2. Get the tree SHA of the latest commit
|
|
160
|
+
const { data: latestCommit } = await this.octokit.git.getCommit({
|
|
161
|
+
owner: this.config.owner,
|
|
162
|
+
repo: this.config.repo,
|
|
163
|
+
commit_sha: latestCommitSha,
|
|
164
|
+
});
|
|
165
|
+
const baseTreeSha = latestCommit.tree.sha;
|
|
166
|
+
// 3. Create a new tree with multiple files
|
|
167
|
+
const treeEntries = changes.map((change) => {
|
|
168
|
+
if (change.content === null) {
|
|
169
|
+
return {
|
|
170
|
+
path: change.path,
|
|
171
|
+
mode: "100644",
|
|
172
|
+
type: "blob",
|
|
173
|
+
sha: null,
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
return {
|
|
177
|
+
path: change.path,
|
|
178
|
+
mode: "100644",
|
|
179
|
+
type: "blob",
|
|
180
|
+
content: JSON.stringify(change.content, null, 2),
|
|
181
|
+
};
|
|
182
|
+
});
|
|
183
|
+
const { data: newTree } = await this.octokit.git.createTree({
|
|
184
|
+
owner: this.config.owner,
|
|
185
|
+
repo: this.config.repo,
|
|
186
|
+
base_tree: baseTreeSha,
|
|
187
|
+
tree: treeEntries,
|
|
188
|
+
});
|
|
189
|
+
// 4. Create a new commit
|
|
190
|
+
const { data: newCommit } = await this.octokit.git.createCommit({
|
|
191
|
+
owner: this.config.owner,
|
|
192
|
+
repo: this.config.repo,
|
|
193
|
+
message,
|
|
194
|
+
tree: newTree.sha,
|
|
195
|
+
parents: [latestCommitSha],
|
|
196
|
+
});
|
|
197
|
+
// 5. Update the reference
|
|
198
|
+
await this.octokit.git.updateRef({
|
|
199
|
+
owner: this.config.owner,
|
|
200
|
+
repo: this.config.repo,
|
|
201
|
+
ref: `heads/${branch}`,
|
|
202
|
+
sha: newCommit.sha,
|
|
203
|
+
});
|
|
204
|
+
// 6. Update cache for all involved files
|
|
205
|
+
const ttl = this.config.cacheTTL ?? this.DEFAULT_TTL;
|
|
206
|
+
for (const entry of newTree.tree) {
|
|
207
|
+
if (entry.path && entry.sha) {
|
|
208
|
+
const change = changes.find((c) => c.path === entry.path);
|
|
209
|
+
if (change) {
|
|
210
|
+
const result = { data: change.content, sha: entry.sha };
|
|
211
|
+
this.cache.set(entry.path, result, ttl);
|
|
212
|
+
this.staleCache.set(entry.path, result);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
return newCommit.sha;
|
|
217
|
+
}
|
|
218
|
+
catch (error) {
|
|
219
|
+
if (error.status === 409) {
|
|
220
|
+
throw new ConcurrencyError("batch-commit");
|
|
221
|
+
}
|
|
222
|
+
throw error;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
async deleteFile(path, message, sha) {
|
|
226
|
+
await this.octokit.repos.deleteFile({
|
|
227
|
+
owner: this.config.owner,
|
|
228
|
+
repo: this.config.repo,
|
|
229
|
+
path,
|
|
230
|
+
message,
|
|
231
|
+
sha,
|
|
232
|
+
});
|
|
233
|
+
this.cache.delete(path);
|
|
234
|
+
this.staleCache.delete(path);
|
|
235
|
+
}
|
|
236
|
+
async listDirectory(path) {
|
|
237
|
+
try {
|
|
238
|
+
const response = await this.octokit.repos.getContent({
|
|
239
|
+
owner: this.config.owner,
|
|
240
|
+
repo: this.config.repo,
|
|
241
|
+
path,
|
|
242
|
+
});
|
|
243
|
+
if (!Array.isArray(response.data)) {
|
|
244
|
+
throw new Error("Path is not a directory");
|
|
245
|
+
}
|
|
246
|
+
return response.data.map((item) => ({
|
|
247
|
+
path: item.path,
|
|
248
|
+
sha: item.sha,
|
|
249
|
+
type: item.type,
|
|
250
|
+
}));
|
|
251
|
+
}
|
|
252
|
+
catch (error) {
|
|
253
|
+
if (error.status === 404) {
|
|
254
|
+
return [];
|
|
255
|
+
}
|
|
256
|
+
throw error;
|
|
257
|
+
}
|
|
258
|
+
}
|
|
138
259
|
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { CommitChange, IStorageProvider, StorageResponse } from "../core/types.js";
|
|
2
|
+
/**
|
|
3
|
+
* A decorator for IStorageProvider that buffers writes in memory.
|
|
4
|
+
* Used during transactions to group multiple operations into a single commit.
|
|
5
|
+
*/
|
|
6
|
+
export declare class TransactionStorageProvider implements IStorageProvider {
|
|
7
|
+
private readonly baseStorage;
|
|
8
|
+
private pendingChanges;
|
|
9
|
+
constructor(baseStorage: IStorageProvider);
|
|
10
|
+
testConnection(): Promise<boolean>;
|
|
11
|
+
exists(path: string): Promise<boolean>;
|
|
12
|
+
readJson<T>(path: string): Promise<StorageResponse<T>>;
|
|
13
|
+
writeJson<T>(path: string, content: T, _message: string, _sha?: string): Promise<string>;
|
|
14
|
+
deleteFile(path: string, _message: string, _sha: string): Promise<void>;
|
|
15
|
+
listDirectory(path: string): Promise<{
|
|
16
|
+
path: string;
|
|
17
|
+
sha: string;
|
|
18
|
+
type: "file" | "dir";
|
|
19
|
+
}[]>;
|
|
20
|
+
commit(changes: CommitChange[], message: string): Promise<string>;
|
|
21
|
+
/**
|
|
22
|
+
* Returns all pending changes in this transaction.
|
|
23
|
+
*/
|
|
24
|
+
getChanges(): CommitChange[];
|
|
25
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A decorator for IStorageProvider that buffers writes in memory.
|
|
3
|
+
* Used during transactions to group multiple operations into a single commit.
|
|
4
|
+
*/
|
|
5
|
+
export class TransactionStorageProvider {
|
|
6
|
+
baseStorage;
|
|
7
|
+
pendingChanges = new Map();
|
|
8
|
+
constructor(baseStorage) {
|
|
9
|
+
this.baseStorage = baseStorage;
|
|
10
|
+
}
|
|
11
|
+
async testConnection() {
|
|
12
|
+
return this.baseStorage.testConnection();
|
|
13
|
+
}
|
|
14
|
+
async exists(path) {
|
|
15
|
+
if (this.pendingChanges.has(path)) {
|
|
16
|
+
return true;
|
|
17
|
+
}
|
|
18
|
+
return this.baseStorage.exists(path);
|
|
19
|
+
}
|
|
20
|
+
async readJson(path) {
|
|
21
|
+
const pending = this.pendingChanges.get(path);
|
|
22
|
+
if (pending !== undefined) {
|
|
23
|
+
return {
|
|
24
|
+
data: pending,
|
|
25
|
+
sha: "transaction-pending-sha",
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
return this.baseStorage.readJson(path);
|
|
29
|
+
}
|
|
30
|
+
async writeJson(path, content, _message, _sha) {
|
|
31
|
+
this.pendingChanges.set(path, content);
|
|
32
|
+
return "transaction-pending-sha";
|
|
33
|
+
}
|
|
34
|
+
async deleteFile(path, _message, _sha) {
|
|
35
|
+
this.pendingChanges.set(path, null);
|
|
36
|
+
}
|
|
37
|
+
async listDirectory(path) {
|
|
38
|
+
// For now, we delegate to base storage.
|
|
39
|
+
// In sharded mode, we might need to merge with pending changes.
|
|
40
|
+
return this.baseStorage.listDirectory(path);
|
|
41
|
+
}
|
|
42
|
+
async commit(changes, message) {
|
|
43
|
+
return this.baseStorage.commit(changes, message);
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Returns all pending changes in this transaction.
|
|
47
|
+
*/
|
|
48
|
+
getChanges() {
|
|
49
|
+
return Array.from(this.pendingChanges.entries()).map(([path, content]) => ({
|
|
50
|
+
path,
|
|
51
|
+
content,
|
|
52
|
+
}));
|
|
53
|
+
}
|
|
54
|
+
}
|
package/dist/ui/collection.d.ts
CHANGED
|
@@ -1,14 +1,22 @@
|
|
|
1
|
-
import { IStorageProvider, Middleware, QueryOptions, Schema } from "../core/types.js";
|
|
1
|
+
import { IStorageProvider, Middleware, QueryOptions, Schema, StorageStrategy, Validator } from "../core/types.js";
|
|
2
2
|
export declare class Collection<T extends Schema> {
|
|
3
3
|
readonly name: string;
|
|
4
4
|
private readonly storage;
|
|
5
|
-
private readonly middleware;
|
|
6
5
|
private lastSha;
|
|
7
6
|
private indexer;
|
|
8
7
|
private items;
|
|
9
8
|
private dataLoaded;
|
|
10
|
-
|
|
9
|
+
private middleware;
|
|
10
|
+
private validator?;
|
|
11
|
+
private strategy;
|
|
12
|
+
private shas;
|
|
13
|
+
constructor(name: string, storage: IStorageProvider, middlewareOrOptions?: Middleware<T>[] | {
|
|
14
|
+
middleware?: Middleware<T>[];
|
|
15
|
+
validator?: Validator<T>;
|
|
16
|
+
strategy?: StorageStrategy;
|
|
17
|
+
});
|
|
11
18
|
private get path();
|
|
19
|
+
private getItemPath;
|
|
12
20
|
create(item: T): Promise<T>;
|
|
13
21
|
find(queryOrPredicate?: ((item: T) => boolean) | QueryOptions<T>): Promise<T[]>;
|
|
14
22
|
private applyQueryOptions;
|
package/dist/ui/collection.js
CHANGED
|
@@ -3,21 +3,42 @@ import { Indexer } from "../core/indexer.js";
|
|
|
3
3
|
export class Collection {
|
|
4
4
|
name;
|
|
5
5
|
storage;
|
|
6
|
-
middleware;
|
|
7
6
|
lastSha;
|
|
8
7
|
indexer = new Indexer();
|
|
9
8
|
items = [];
|
|
10
9
|
dataLoaded = false;
|
|
11
|
-
|
|
10
|
+
middleware;
|
|
11
|
+
validator;
|
|
12
|
+
strategy;
|
|
13
|
+
shas = new Map(); // path -> sha for sharded mode
|
|
14
|
+
constructor(name, storage, middlewareOrOptions) {
|
|
12
15
|
this.name = name;
|
|
13
16
|
this.storage = storage;
|
|
14
|
-
|
|
17
|
+
if (Array.isArray(middlewareOrOptions)) {
|
|
18
|
+
this.middleware = middlewareOrOptions;
|
|
19
|
+
this.strategy = "single-file";
|
|
20
|
+
}
|
|
21
|
+
else {
|
|
22
|
+
this.middleware = middlewareOrOptions?.middleware || [];
|
|
23
|
+
this.validator = middlewareOrOptions?.validator;
|
|
24
|
+
this.strategy = middlewareOrOptions?.strategy || "single-file";
|
|
25
|
+
}
|
|
15
26
|
}
|
|
16
27
|
get path() {
|
|
17
|
-
return `${this.name}.json`;
|
|
28
|
+
return this.strategy === "sharded" ? `${this.name}/` : `${this.name}.json`;
|
|
29
|
+
}
|
|
30
|
+
getItemPath(id) {
|
|
31
|
+
return this.strategy === "sharded"
|
|
32
|
+
? `${this.name}/${id}.json`
|
|
33
|
+
: `${this.name}.json`;
|
|
18
34
|
}
|
|
19
35
|
async create(item) {
|
|
20
36
|
let finalItem = item;
|
|
37
|
+
// 1. Validation
|
|
38
|
+
if (this.validator) {
|
|
39
|
+
finalItem = await this.validator.validate(finalItem);
|
|
40
|
+
}
|
|
41
|
+
// 2. Middleware
|
|
21
42
|
const context = {
|
|
22
43
|
collection: this.name,
|
|
23
44
|
operation: "create",
|
|
@@ -27,6 +48,22 @@ export class Collection {
|
|
|
27
48
|
finalItem = await mw.beforeSave(finalItem, context);
|
|
28
49
|
}
|
|
29
50
|
}
|
|
51
|
+
if (this.strategy === "sharded") {
|
|
52
|
+
const itemPath = this.getItemPath(finalItem.id);
|
|
53
|
+
const sha = await this.storage.writeJson(itemPath, finalItem, `Create item ${finalItem.id} in ${this.name}`, this.shas.get(itemPath));
|
|
54
|
+
this.shas.set(itemPath, sha);
|
|
55
|
+
if (this.dataLoaded) {
|
|
56
|
+
const index = this.items.findIndex((i) => i.id === finalItem.id);
|
|
57
|
+
if (index > -1) {
|
|
58
|
+
this.items[index] = finalItem;
|
|
59
|
+
}
|
|
60
|
+
else {
|
|
61
|
+
this.items.push(finalItem);
|
|
62
|
+
}
|
|
63
|
+
this.indexer.add(finalItem);
|
|
64
|
+
}
|
|
65
|
+
return finalItem;
|
|
66
|
+
}
|
|
30
67
|
let items = [];
|
|
31
68
|
if (this.dataLoaded) {
|
|
32
69
|
items = [...this.items];
|
|
@@ -92,9 +129,21 @@ export class Collection {
|
|
|
92
129
|
}
|
|
93
130
|
return this.items;
|
|
94
131
|
}
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
132
|
+
if (this.strategy === "sharded") {
|
|
133
|
+
const files = await this.storage.listDirectory(this.name);
|
|
134
|
+
items = await Promise.all(files
|
|
135
|
+
.filter((f) => f.type === "file" && f.path.endsWith(".json"))
|
|
136
|
+
.map(async (file) => {
|
|
137
|
+
const response = await this.storage.readJson(file.path);
|
|
138
|
+
this.shas.set(file.path, response.sha);
|
|
139
|
+
return response.data;
|
|
140
|
+
}));
|
|
141
|
+
}
|
|
142
|
+
else {
|
|
143
|
+
const response = await this.storage.readJson(this.path);
|
|
144
|
+
this.lastSha = response.sha;
|
|
145
|
+
items = response.data;
|
|
146
|
+
}
|
|
98
147
|
items = await Promise.all(items.map(async (item) => {
|
|
99
148
|
let currentItem = item;
|
|
100
149
|
for (const mw of this.middleware) {
|
|
@@ -139,6 +188,9 @@ export class Collection {
|
|
|
139
188
|
case "lte":
|
|
140
189
|
return val <= filter.value;
|
|
141
190
|
case "contains":
|
|
191
|
+
if (Array.isArray(val)) {
|
|
192
|
+
return val.includes(filter.value);
|
|
193
|
+
}
|
|
142
194
|
return (typeof val === "string" && val.includes(filter.value));
|
|
143
195
|
case "in":
|
|
144
196
|
return Array.isArray(filter.value) && filter.value.includes(val);
|
|
@@ -174,6 +226,31 @@ export class Collection {
|
|
|
174
226
|
const results = this.indexer.query("id", id);
|
|
175
227
|
return results && results.length > 0 ? results[0] : null;
|
|
176
228
|
}
|
|
229
|
+
if (this.strategy === "sharded") {
|
|
230
|
+
try {
|
|
231
|
+
const itemPath = this.getItemPath(id);
|
|
232
|
+
const response = await this.storage.readJson(itemPath);
|
|
233
|
+
this.shas.set(itemPath, response.sha);
|
|
234
|
+
let finalItem = response.data;
|
|
235
|
+
const context = {
|
|
236
|
+
collection: this.name,
|
|
237
|
+
operation: "read",
|
|
238
|
+
};
|
|
239
|
+
for (const mw of this.middleware) {
|
|
240
|
+
if (mw.afterRead) {
|
|
241
|
+
finalItem = await mw.afterRead(finalItem, context);
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
// If data is not fully loaded, we don't update this.items yet to avoid partial consistency
|
|
245
|
+
// But if someone calls find() later it will load everything.
|
|
246
|
+
return finalItem;
|
|
247
|
+
}
|
|
248
|
+
catch (error) {
|
|
249
|
+
if (error.status === 404)
|
|
250
|
+
return null;
|
|
251
|
+
throw error;
|
|
252
|
+
}
|
|
253
|
+
}
|
|
177
254
|
const items = await this.find();
|
|
178
255
|
return items.find((item) => item.id === id) || null;
|
|
179
256
|
}
|
|
@@ -186,6 +263,11 @@ export class Collection {
|
|
|
186
263
|
const originalItem = items[index];
|
|
187
264
|
items[index] = { ...items[index], ...updates };
|
|
188
265
|
let finalItem = items[index];
|
|
266
|
+
// 1. Validation
|
|
267
|
+
if (this.validator) {
|
|
268
|
+
finalItem = await this.validator.validate(finalItem);
|
|
269
|
+
}
|
|
270
|
+
// 2. Middleware
|
|
189
271
|
const context = {
|
|
190
272
|
collection: this.name,
|
|
191
273
|
operation: "update",
|
|
@@ -196,14 +278,29 @@ export class Collection {
|
|
|
196
278
|
}
|
|
197
279
|
}
|
|
198
280
|
items[index] = finalItem;
|
|
199
|
-
|
|
200
|
-
|
|
281
|
+
if (this.strategy === "sharded") {
|
|
282
|
+
const itemPath = this.getItemPath(id);
|
|
283
|
+
try {
|
|
284
|
+
const sha = await this.storage.writeJson(itemPath, finalItem, `Update item ${id} in ${this.name}`, this.shas.get(itemPath));
|
|
285
|
+
this.shas.set(itemPath, sha);
|
|
286
|
+
}
|
|
287
|
+
catch (error) {
|
|
288
|
+
if (error instanceof ConcurrencyError) {
|
|
289
|
+
this.dataLoaded = false;
|
|
290
|
+
}
|
|
291
|
+
throw error;
|
|
292
|
+
}
|
|
201
293
|
}
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
this.
|
|
294
|
+
else {
|
|
295
|
+
try {
|
|
296
|
+
this.lastSha = await this.storage.writeJson(this.path, items, `Update item ${id} in ${this.name}`, this.lastSha);
|
|
297
|
+
}
|
|
298
|
+
catch (error) {
|
|
299
|
+
if (error instanceof ConcurrencyError) {
|
|
300
|
+
this.dataLoaded = false;
|
|
301
|
+
}
|
|
302
|
+
throw error;
|
|
205
303
|
}
|
|
206
|
-
throw error;
|
|
207
304
|
}
|
|
208
305
|
if (this.dataLoaded) {
|
|
209
306
|
this.items = items;
|
|
@@ -219,14 +316,29 @@ export class Collection {
|
|
|
219
316
|
}
|
|
220
317
|
const itemToDelete = items[index];
|
|
221
318
|
const filtered = items.filter((_, i) => i !== index);
|
|
222
|
-
|
|
223
|
-
|
|
319
|
+
if (this.strategy === "sharded") {
|
|
320
|
+
const itemPath = this.getItemPath(id);
|
|
321
|
+
const sha = this.shas.get(itemPath);
|
|
322
|
+
if (!sha) {
|
|
323
|
+
// We need the SHA to delete. If we don't have it, we must fetch it.
|
|
324
|
+
const response = await this.storage.readJson(itemPath);
|
|
325
|
+
await this.storage.deleteFile(itemPath, `Delete item ${id} from ${this.name}`, response.sha);
|
|
326
|
+
}
|
|
327
|
+
else {
|
|
328
|
+
await this.storage.deleteFile(itemPath, `Delete item ${id} from ${this.name}`, sha);
|
|
329
|
+
}
|
|
330
|
+
this.shas.delete(itemPath);
|
|
224
331
|
}
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
this.
|
|
332
|
+
else {
|
|
333
|
+
try {
|
|
334
|
+
this.lastSha = await this.storage.writeJson(this.path, filtered, `Delete item ${id} from ${this.name}`, this.lastSha);
|
|
335
|
+
}
|
|
336
|
+
catch (error) {
|
|
337
|
+
if (error instanceof ConcurrencyError) {
|
|
338
|
+
this.dataLoaded = false;
|
|
339
|
+
}
|
|
340
|
+
throw error;
|
|
228
341
|
}
|
|
229
|
-
throw error;
|
|
230
342
|
}
|
|
231
343
|
if (this.dataLoaded) {
|
|
232
344
|
this.items = filtered;
|
package/dist/ui/github-db.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { GitHubDBConfig, IStorageProvider, Middleware, Schema } from "../core/types.js";
|
|
1
|
+
import { GitHubDBConfig, IStorageProvider, Middleware, Schema, Validator } from "../core/types.js";
|
|
2
2
|
import { Collection } from "./collection.js";
|
|
3
|
+
import { Transaction } from "./transaction.js";
|
|
3
4
|
export declare class GitHubDB {
|
|
4
5
|
readonly config: GitHubDBConfig;
|
|
5
6
|
readonly storage: IStorageProvider;
|
|
@@ -7,5 +8,7 @@ export declare class GitHubDB {
|
|
|
7
8
|
connect(): Promise<boolean>;
|
|
8
9
|
collection<T extends Schema>(name: string, options?: {
|
|
9
10
|
middleware?: Middleware<T>[];
|
|
11
|
+
validator?: Validator<T>;
|
|
10
12
|
}): Collection<T>;
|
|
13
|
+
transaction(fn: (tx: Transaction) => Promise<void>, message?: string): Promise<string>;
|
|
11
14
|
}
|
package/dist/ui/github-db.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { GitHubStorageProvider } from "../infrastructure/github-storage.js";
|
|
2
2
|
import { Collection } from "./collection.js";
|
|
3
|
+
import { Transaction } from "./transaction.js";
|
|
3
4
|
export class GitHubDB {
|
|
4
5
|
config;
|
|
5
6
|
storage;
|
|
@@ -20,6 +21,11 @@ export class GitHubDB {
|
|
|
20
21
|
return this.storage.testConnection();
|
|
21
22
|
}
|
|
22
23
|
collection(name, options = {}) {
|
|
23
|
-
return new Collection(name, this.storage, options
|
|
24
|
+
return new Collection(name, this.storage, options);
|
|
25
|
+
}
|
|
26
|
+
async transaction(fn, message = "Transaction commit") {
|
|
27
|
+
const tx = new Transaction(this.storage);
|
|
28
|
+
await fn(tx);
|
|
29
|
+
return tx.commit(message);
|
|
24
30
|
}
|
|
25
31
|
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { IStorageProvider, Middleware, Schema, Validator } from "../core/types.js";
|
|
2
|
+
import { Collection } from "./collection.js";
|
|
3
|
+
/**
|
|
4
|
+
* Handles atomic batch operations across multiple collections.
|
|
5
|
+
*/
|
|
6
|
+
export declare class Transaction {
|
|
7
|
+
private readonly baseStorage;
|
|
8
|
+
private txStorage;
|
|
9
|
+
constructor(baseStorage: IStorageProvider);
|
|
10
|
+
/**
|
|
11
|
+
* Returns a collection instance bound to this transaction.
|
|
12
|
+
* Operations on this collection will be buffered until the transaction is committed.
|
|
13
|
+
*/
|
|
14
|
+
collection<T extends Schema>(name: string, options?: {
|
|
15
|
+
middleware?: Middleware<T>[];
|
|
16
|
+
validator?: Validator<T>;
|
|
17
|
+
}): Collection<T>;
|
|
18
|
+
/**
|
|
19
|
+
* Commits all buffered changes in the transaction to GitHub in a single commit.
|
|
20
|
+
* @internal
|
|
21
|
+
*/
|
|
22
|
+
commit(message: string): Promise<string>;
|
|
23
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { TransactionStorageProvider } from "../infrastructure/transaction-storage.js";
|
|
2
|
+
import { Collection } from "./collection.js";
|
|
3
|
+
/**
|
|
4
|
+
* Handles atomic batch operations across multiple collections.
|
|
5
|
+
*/
|
|
6
|
+
export class Transaction {
|
|
7
|
+
baseStorage;
|
|
8
|
+
txStorage;
|
|
9
|
+
constructor(baseStorage) {
|
|
10
|
+
this.baseStorage = baseStorage;
|
|
11
|
+
this.txStorage = new TransactionStorageProvider(baseStorage);
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Returns a collection instance bound to this transaction.
|
|
15
|
+
* Operations on this collection will be buffered until the transaction is committed.
|
|
16
|
+
*/
|
|
17
|
+
collection(name, options = {}) {
|
|
18
|
+
return new Collection(name, this.txStorage, options);
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Commits all buffered changes in the transaction to GitHub in a single commit.
|
|
22
|
+
* @internal
|
|
23
|
+
*/
|
|
24
|
+
async commit(message) {
|
|
25
|
+
const changes = this.txStorage.getChanges();
|
|
26
|
+
if (changes.length === 0) {
|
|
27
|
+
return "";
|
|
28
|
+
}
|
|
29
|
+
return this.txStorage.commit(changes, message);
|
|
30
|
+
}
|
|
31
|
+
}
|