@an5/adapters 0.2.2 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -5
- package/package.json +3 -1
- package/typescript/an5Adapter.js +2 -2
- package/typescript/base/uuid.d.ts +5 -0
- package/typescript/base/uuid.js +17 -0
- package/typescript/googlesheets/adapter.d.ts +1 -1
- package/typescript/googlesheets/adapter.js +19 -8
- package/typescript/googlesheets/config.d.ts +5 -1
- package/typescript/googlesheets/config.js +2 -1
- package/typescript/googlesheets/helpers.js +28 -9
- package/typescript/googlesheets/parseConnectionString.js +4 -0
- package/typescript/googlesheets/retry.d.ts +23 -1
- package/typescript/googlesheets/retry.js +87 -11
- package/typescript/googlesheets/tableClient.d.ts +1 -0
- package/typescript/googlesheets/tableClient.js +59 -42
- package/typescript/src/an5Adapter.ts +2 -2
- package/typescript/src/base/uuid.ts +14 -0
- package/typescript/src/googlesheets/adapter.ts +25 -8
- package/typescript/src/googlesheets/config.ts +4 -1
- package/typescript/src/googlesheets/helpers.ts +25 -6
- package/typescript/src/googlesheets/parseConnectionString.ts +2 -0
- package/typescript/src/googlesheets/retry.ts +104 -10
- package/typescript/src/googlesheets/tableClient.ts +68 -48
package/README.md
CHANGED
|
@@ -19,7 +19,7 @@ Adapters are runtime packages only. They do not import generated `an5Client` art
|
|
|
19
19
|
### TypeScript
|
|
20
20
|
|
|
21
21
|
```bash
|
|
22
|
-
npm install an5
|
|
22
|
+
npm install @an5/adapters
|
|
23
23
|
```
|
|
24
24
|
|
|
25
25
|
### Python
|
|
@@ -39,7 +39,7 @@ dotnet add package An5Adapters
|
|
|
39
39
|
### TypeScript
|
|
40
40
|
|
|
41
41
|
```typescript
|
|
42
|
-
import { createAn5Adapter, setAdapterMetadata } from 'an5
|
|
42
|
+
import { createAn5Adapter, setAdapterMetadata } from '@an5/adapters';
|
|
43
43
|
|
|
44
44
|
setAdapterMetadata({
|
|
45
45
|
modelToTable: { User: 'dbo.users' },
|
|
@@ -120,7 +120,7 @@ db.Transaction(tx => {
|
|
|
120
120
|
### Google Sheets
|
|
121
121
|
|
|
122
122
|
```typescript
|
|
123
|
-
import { createAn5SheetsAdapter } from 'an5
|
|
123
|
+
import { createAn5SheetsAdapter } from '@an5/adapters';
|
|
124
124
|
|
|
125
125
|
const db = createAn5SheetsAdapter({
|
|
126
126
|
spreadsheetId: '1BxiMVs0XRA5nFMdKvBdBZjgmUUqptlbs74OgVE2upms',
|
|
@@ -157,10 +157,30 @@ await db.table('users').clear();
|
|
|
157
157
|
await db.table('users').deleteAll();
|
|
158
158
|
```
|
|
159
159
|
|
|
160
|
+
### Browser Mode (Google Sheets)
|
|
161
|
+
|
|
162
|
+
For browser environments (React, Vue, Svelte, Next.js client components), import from `@an5/adapters/browser` and configure `accessToken` or `apiKey`. In browser mode, the adapter uses native `fetch()` without importing Node.js modules (`googleapis`, `crypto`, `fs`).
|
|
163
|
+
|
|
164
|
+
```typescript
|
|
165
|
+
import { createAn5SheetsAdapter } from '@an5/adapters/browser';
|
|
166
|
+
|
|
167
|
+
// Option 1: OAuth2 Access Token (e.g. from Firebase Auth or Google Sign-In)
|
|
168
|
+
const db = createAn5SheetsAdapter({
|
|
169
|
+
spreadsheetId: '1BxiMVs0XRA5nFMdKvBdBZjgmUUqptlbs74OgVE2upms',
|
|
170
|
+
accessToken: userOAuthAccessToken,
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
// Option 2: Google API Key
|
|
174
|
+
const db2 = createAn5SheetsAdapter({
|
|
175
|
+
spreadsheetId: '1BxiMVs0XRA5nFMdKvBdBZjgmUUqptlbs74OgVE2upms',
|
|
176
|
+
apiKey: 'AIzaSy...',
|
|
177
|
+
});
|
|
178
|
+
```
|
|
179
|
+
|
|
160
180
|
### Integrated factory (auto-detect adapter)
|
|
161
181
|
|
|
162
182
|
```typescript
|
|
163
|
-
import { createAn5Adapter, createAdapter, An5Adapter } from 'an5
|
|
183
|
+
import { createAn5Adapter, createAdapter, An5Adapter } from '@an5/adapters';
|
|
164
184
|
|
|
165
185
|
// Auto-detects from connection string
|
|
166
186
|
const sqlDb = createAn5Adapter({ connectionString: 'sqlserver://localhost:1433;database=mydb;user=sa;password=pass' });
|
|
@@ -197,7 +217,7 @@ const sheetsDb3 = new An5Adapter({
|
|
|
197
217
|
Use the package root for normal applications:
|
|
198
218
|
|
|
199
219
|
```typescript
|
|
200
|
-
import { createAn5Adapter, createAn5SheetsAdapter } from 'an5
|
|
220
|
+
import { createAn5Adapter, createAn5SheetsAdapter } from '@an5/adapters';
|
|
201
221
|
```
|
|
202
222
|
|
|
203
223
|
Provider folders are still available to source-level consumers through `typescript/*`, but the public factory in `typescript/an5Adapter.ts` is the preferred entry point. The old `unified.ts` entry point has been removed because the factory now lives directly in `An5Adapter`.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@an5/adapters",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.3",
|
|
4
4
|
"description": "Standalone language adapters for AN5 ORM - TypeScript, Python, .NET query execution.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"main": "typescript/index.js",
|
|
@@ -14,6 +14,8 @@
|
|
|
14
14
|
"./dotnet": "./dotnet/An5Adapter.cs",
|
|
15
15
|
"./config": "./typescript/config.js"
|
|
16
16
|
},
|
|
17
|
+
"author": "an5ORM",
|
|
18
|
+
"homepage": "https://github.com/an5ORM/an5",
|
|
17
19
|
"repository": {
|
|
18
20
|
"type": "git",
|
|
19
21
|
"url": "https://github.com/an5ORM/an5Adapters.git"
|
package/typescript/an5Adapter.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.createAdapter = exports.AdapterTableClient = exports.An5AdapterTx = exports.An5Adapter = exports.setAdapterMetadata = void 0;
|
|
4
4
|
exports.createAn5Adapter = createAn5Adapter;
|
|
5
|
-
const
|
|
5
|
+
const uuid_1 = require("./base/uuid");
|
|
6
6
|
const googlesheets_1 = require("./googlesheets");
|
|
7
7
|
const parseConnectionString_1 = require("./googlesheets/parseConnectionString");
|
|
8
8
|
const sql_1 = require("./base/sql");
|
|
@@ -304,7 +304,7 @@ class AdapterTableClient {
|
|
|
304
304
|
const normalizedType = String(rawType).toLowerCase();
|
|
305
305
|
const isStringType = ['string', 'uuid', 'uniqueidentifier', 'nvarchar', 'varchar', 'text'].includes(normalizedType);
|
|
306
306
|
if (isStringType && !data[idFieldName])
|
|
307
|
-
data[idFieldName] = (0,
|
|
307
|
+
data[idFieldName] = (0, uuid_1.generateUUID)();
|
|
308
308
|
}
|
|
309
309
|
const cols = Object.keys(data).filter(k => data[k] !== undefined);
|
|
310
310
|
const params = {};
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.generateUUID = generateUUID;
|
|
4
|
+
/**
|
|
5
|
+
* Cross-platform UUID generator safe for both Browser and Node.js environments.
|
|
6
|
+
* Uses crypto.randomUUID() when available, with a standard RFC4122 v4 fallback.
|
|
7
|
+
*/
|
|
8
|
+
function generateUUID() {
|
|
9
|
+
if (typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function') {
|
|
10
|
+
return crypto.randomUUID();
|
|
11
|
+
}
|
|
12
|
+
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
|
|
13
|
+
const r = (Math.random() * 16) | 0;
|
|
14
|
+
const v = c === 'x' ? r : (r & 0x3) | 0x8;
|
|
15
|
+
return v.toString(16);
|
|
16
|
+
});
|
|
17
|
+
}
|
|
@@ -7,7 +7,7 @@ export declare class An5SheetsAdapter {
|
|
|
7
7
|
config: ReturnType<typeof resolveConfig>;
|
|
8
8
|
private sheetsCache;
|
|
9
9
|
constructor(config: An5SheetsAdapterConfig);
|
|
10
|
-
private get
|
|
10
|
+
private get isFetchMode();
|
|
11
11
|
getSheets(): Promise<any>;
|
|
12
12
|
private getApi;
|
|
13
13
|
invalidateCache(): void;
|
|
@@ -6,12 +6,19 @@ const config_1 = require("./config");
|
|
|
6
6
|
const sqlExecutor_1 = require("./sqlExecutor");
|
|
7
7
|
const tableClient_1 = require("./tableClient");
|
|
8
8
|
const retry_1 = require("./retry");
|
|
9
|
-
// ─── Fetch-based API proxy for OAuth Access Token (browser-compatible)
|
|
10
|
-
function createFetchApi(spreadsheetId, accessToken) {
|
|
9
|
+
// ─── Fetch-based API proxy for OAuth Access Token / API Key (browser-compatible) ──
|
|
10
|
+
function createFetchApi(spreadsheetId, accessToken, apiKey) {
|
|
11
11
|
const base = `https://sheets.googleapis.com/v4/spreadsheets/${spreadsheetId}`;
|
|
12
|
-
const headers = {
|
|
12
|
+
const headers = { 'Content-Type': 'application/json' };
|
|
13
|
+
if (accessToken) {
|
|
14
|
+
headers['Authorization'] = `Bearer ${accessToken}`;
|
|
15
|
+
}
|
|
13
16
|
async function request(method, path, body) {
|
|
14
|
-
|
|
17
|
+
let url = path.startsWith('http') ? path : `${base}${path}`;
|
|
18
|
+
if (apiKey) {
|
|
19
|
+
const sep = url.includes('?') ? '&' : '?';
|
|
20
|
+
url += `${sep}key=${encodeURIComponent(apiKey)}`;
|
|
21
|
+
}
|
|
15
22
|
const opts = { method, headers };
|
|
16
23
|
if (body)
|
|
17
24
|
opts.body = JSON.stringify(body);
|
|
@@ -73,16 +80,20 @@ class An5SheetsAdapter {
|
|
|
73
80
|
this.sheetsCache = null;
|
|
74
81
|
this.config = (0, config_1.resolveConfig)(config);
|
|
75
82
|
}
|
|
76
|
-
get
|
|
77
|
-
return !!this.config.accessToken;
|
|
83
|
+
get isFetchMode() {
|
|
84
|
+
return !!this.config.accessToken || !!this.config.apiKey;
|
|
78
85
|
}
|
|
79
86
|
async getSheets() {
|
|
80
|
-
if (this.
|
|
87
|
+
if (this.isFetchMode) {
|
|
81
88
|
if (!this.fetchApi) {
|
|
82
|
-
this.fetchApi = createFetchApi(this.config.spreadsheetId, this.config.accessToken);
|
|
89
|
+
this.fetchApi = createFetchApi(this.config.spreadsheetId, this.config.accessToken, this.config.apiKey);
|
|
83
90
|
}
|
|
84
91
|
return this.fetchApi;
|
|
85
92
|
}
|
|
93
|
+
const isBrowser = typeof window !== 'undefined' && typeof window.document !== 'undefined';
|
|
94
|
+
if (isBrowser) {
|
|
95
|
+
throw new Error('Google Sheets adapter in browser mode requires "accessToken" or "apiKey" configuration.');
|
|
96
|
+
}
|
|
86
97
|
if (!this.sheets) {
|
|
87
98
|
const { google } = await import('googleapis');
|
|
88
99
|
const auth = new google.auth.JWT({
|
|
@@ -11,11 +11,14 @@ export interface An5SheetsAdapterConfig {
|
|
|
11
11
|
* When provided, the adapter uses raw fetch() instead of googleapis JWT, making it compatible
|
|
12
12
|
* with browser environments. Mutually exclusive with clientEmail/privateKey/credentials. */
|
|
13
13
|
accessToken?: string;
|
|
14
|
+
/** Google API key for browser-based API access. */
|
|
15
|
+
apiKey?: string;
|
|
14
16
|
}
|
|
15
17
|
export declare function normalizeKey(key: string): string;
|
|
16
18
|
export declare function resolveConfig(config: An5SheetsAdapterConfig): {
|
|
17
19
|
spreadsheetId: string;
|
|
18
|
-
accessToken: string;
|
|
20
|
+
accessToken: string | undefined;
|
|
21
|
+
apiKey: string | undefined;
|
|
19
22
|
sheetMapping: Record<string, string> | undefined;
|
|
20
23
|
clientEmail?: undefined;
|
|
21
24
|
privateKey?: undefined;
|
|
@@ -25,4 +28,5 @@ export declare function resolveConfig(config: An5SheetsAdapterConfig): {
|
|
|
25
28
|
privateKey: string;
|
|
26
29
|
sheetMapping: Record<string, string> | undefined;
|
|
27
30
|
accessToken?: undefined;
|
|
31
|
+
apiKey?: undefined;
|
|
28
32
|
};
|
|
@@ -12,10 +12,11 @@ function normalizeKey(key) {
|
|
|
12
12
|
return `-----BEGIN PRIVATE KEY-----\n${lines}\n-----END PRIVATE KEY-----`;
|
|
13
13
|
}
|
|
14
14
|
function resolveConfig(config) {
|
|
15
|
-
if (config.accessToken) {
|
|
15
|
+
if (config.accessToken || config.apiKey) {
|
|
16
16
|
return {
|
|
17
17
|
spreadsheetId: config.spreadsheetId,
|
|
18
18
|
accessToken: config.accessToken,
|
|
19
|
+
apiKey: config.apiKey,
|
|
19
20
|
sheetMapping: config.sheetMapping,
|
|
20
21
|
};
|
|
21
22
|
}
|
|
@@ -45,6 +45,12 @@ function matchWhere(row, where) {
|
|
|
45
45
|
return false;
|
|
46
46
|
continue;
|
|
47
47
|
}
|
|
48
|
+
if (key === 'NOT') {
|
|
49
|
+
const notClauses = Array.isArray(value) ? value : [value];
|
|
50
|
+
if (notClauses.some((v) => matchWhere(row, v)))
|
|
51
|
+
return false;
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
48
54
|
const cellVal = row[key];
|
|
49
55
|
if (value === null) {
|
|
50
56
|
if (cellVal !== null && cellVal !== undefined)
|
|
@@ -52,22 +58,30 @@ function matchWhere(row, where) {
|
|
|
52
58
|
}
|
|
53
59
|
else if (typeof value === 'object' && !(value instanceof Date)) {
|
|
54
60
|
const v = value;
|
|
61
|
+
const caseInsensitive = v.mode === 'insensitive';
|
|
62
|
+
const normalize = (s) => caseInsensitive ? String(s ?? '').toLowerCase() : s;
|
|
55
63
|
if (v.not !== undefined) {
|
|
56
64
|
if (v.not === null) {
|
|
57
65
|
if (cellVal === null || cellVal === undefined)
|
|
58
66
|
return false;
|
|
59
67
|
}
|
|
60
|
-
else if (cellVal == v.not)
|
|
68
|
+
else if (normalize(cellVal) == normalize(v.not))
|
|
61
69
|
return false;
|
|
62
70
|
}
|
|
63
|
-
if (v.equals !== undefined && cellVal != v.equals)
|
|
64
|
-
return false;
|
|
65
|
-
if (v.contains !== undefined && (!cellVal || !String(cellVal).includes(v.contains)))
|
|
66
|
-
return false;
|
|
67
|
-
if (v.startsWith !== undefined && (!cellVal || !String(cellVal).startsWith(v.startsWith)))
|
|
68
|
-
return false;
|
|
69
|
-
if (v.endsWith !== undefined && (!cellVal || !String(cellVal).endsWith(v.endsWith)))
|
|
71
|
+
if (v.equals !== undefined && normalize(cellVal) != normalize(v.equals))
|
|
70
72
|
return false;
|
|
73
|
+
if (v.contains !== undefined) {
|
|
74
|
+
if (!cellVal || !normalize(String(cellVal)).includes(normalize(v.contains)))
|
|
75
|
+
return false;
|
|
76
|
+
}
|
|
77
|
+
if (v.startsWith !== undefined) {
|
|
78
|
+
if (!cellVal || !normalize(String(cellVal)).startsWith(normalize(v.startsWith)))
|
|
79
|
+
return false;
|
|
80
|
+
}
|
|
81
|
+
if (v.endsWith !== undefined) {
|
|
82
|
+
if (!cellVal || !normalize(String(cellVal)).endsWith(normalize(v.endsWith)))
|
|
83
|
+
return false;
|
|
84
|
+
}
|
|
71
85
|
if (v.gte !== undefined && !(Number(cellVal) >= Number(v.gte)))
|
|
72
86
|
return false;
|
|
73
87
|
if (v.lte !== undefined && !(Number(cellVal) <= Number(v.lte)))
|
|
@@ -77,11 +91,16 @@ function matchWhere(row, where) {
|
|
|
77
91
|
if (v.lt !== undefined && !(Number(cellVal) < Number(v.lt)))
|
|
78
92
|
return false;
|
|
79
93
|
if (v.in !== undefined) {
|
|
80
|
-
if (!Array.isArray(v.in) || !v.in.
|
|
94
|
+
if (!Array.isArray(v.in) || !v.in.some((item) => item == cellVal))
|
|
95
|
+
return false;
|
|
96
|
+
}
|
|
97
|
+
if (v.notIn !== undefined) {
|
|
98
|
+
if (Array.isArray(v.notIn) && v.notIn.some((item) => item == cellVal))
|
|
81
99
|
return false;
|
|
82
100
|
}
|
|
83
101
|
}
|
|
84
102
|
else {
|
|
103
|
+
// loose equality to handle string/number coercion from sheet cells
|
|
85
104
|
if (cellVal != value)
|
|
86
105
|
return false;
|
|
87
106
|
}
|
|
@@ -31,6 +31,10 @@ function parseSheetsConnectionString(url) {
|
|
|
31
31
|
result.clientEmail = config.clientemail;
|
|
32
32
|
if (config.privatekey)
|
|
33
33
|
result.privateKey = config.privatekey;
|
|
34
|
+
if (config.accesstoken)
|
|
35
|
+
result.accessToken = config.accesstoken;
|
|
36
|
+
if (config.apikey)
|
|
37
|
+
result.apiKey = config.apikey;
|
|
34
38
|
if (config.sheetmapping) {
|
|
35
39
|
const mapping = {};
|
|
36
40
|
for (const pair of config.sheetmapping.split(',')) {
|
|
@@ -1,3 +1,25 @@
|
|
|
1
|
-
export declare const MAX_RETRIES =
|
|
1
|
+
export declare const MAX_RETRIES = 5;
|
|
2
2
|
export declare const BASE_DELAY = 1000;
|
|
3
|
+
export declare const MAX_DELAY = 32000;
|
|
4
|
+
export declare const MAX_CONCURRENCY = 4;
|
|
5
|
+
export declare const RATE_LIMIT_RPS = 0.9;
|
|
6
|
+
declare class TokenBucket {
|
|
7
|
+
private tokens;
|
|
8
|
+
private lastRefill;
|
|
9
|
+
private readonly ratePerMs;
|
|
10
|
+
private readonly capacity;
|
|
11
|
+
constructor(ratePerSecond: number, capacity: number);
|
|
12
|
+
/** Wait until a token is available, then consume one. */
|
|
13
|
+
acquire(): Promise<void>;
|
|
14
|
+
}
|
|
15
|
+
declare class ConcurrencyLimiter {
|
|
16
|
+
private readonly max;
|
|
17
|
+
private running;
|
|
18
|
+
private queue;
|
|
19
|
+
constructor(max: number);
|
|
20
|
+
acquire(): Promise<void>;
|
|
21
|
+
release(): void;
|
|
22
|
+
}
|
|
3
23
|
export declare function withRetry<T>(fn: () => Promise<T>, attempt?: number): Promise<T>;
|
|
24
|
+
/** Expose for testing / custom configurations */
|
|
25
|
+
export { TokenBucket, ConcurrencyLimiter };
|
|
@@ -1,19 +1,83 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
// ─── Retry helper
|
|
2
|
+
// ─── Rate Limit & Retry helper ─────────────────────────────────────────────────
|
|
3
|
+
//
|
|
4
|
+
// Google Sheets API quota (default): 100 requests per 100 seconds per project.
|
|
5
|
+
// Strategy:
|
|
6
|
+
// 1. Token-bucket rate limiter — proactively throttles before hitting quota
|
|
7
|
+
// 2. Concurrency limiter — max N parallel requests to avoid burst
|
|
8
|
+
// 3. Exponential backoff + jitter — reactive recovery when 429/403/5xx hit
|
|
3
9
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
-
exports.BASE_DELAY = exports.MAX_RETRIES = void 0;
|
|
10
|
+
exports.ConcurrencyLimiter = exports.TokenBucket = exports.RATE_LIMIT_RPS = exports.MAX_CONCURRENCY = exports.MAX_DELAY = exports.BASE_DELAY = exports.MAX_RETRIES = void 0;
|
|
5
11
|
exports.withRetry = withRetry;
|
|
6
|
-
|
|
7
|
-
exports.
|
|
8
|
-
|
|
12
|
+
// ─── Config ────────────────────────────────────────────────────────────────────
|
|
13
|
+
exports.MAX_RETRIES = 5;
|
|
14
|
+
exports.BASE_DELAY = 1000; // ms
|
|
15
|
+
exports.MAX_DELAY = 32000; // ms cap for backoff
|
|
16
|
+
exports.MAX_CONCURRENCY = 4; // parallel in-flight requests
|
|
17
|
+
exports.RATE_LIMIT_RPS = 0.9; // tokens/second (stay under 1 req/s safely)
|
|
18
|
+
// ─── Token Bucket ──────────────────────────────────────────────────────────────
|
|
19
|
+
class TokenBucket {
|
|
20
|
+
constructor(ratePerSecond, capacity) {
|
|
21
|
+
this.ratePerMs = ratePerSecond / 1000;
|
|
22
|
+
this.capacity = capacity;
|
|
23
|
+
this.tokens = capacity;
|
|
24
|
+
this.lastRefill = Date.now();
|
|
25
|
+
}
|
|
26
|
+
/** Wait until a token is available, then consume one. */
|
|
27
|
+
async acquire() {
|
|
28
|
+
while (true) {
|
|
29
|
+
const now = Date.now();
|
|
30
|
+
const elapsed = now - this.lastRefill;
|
|
31
|
+
this.tokens = Math.min(this.capacity, this.tokens + elapsed * this.ratePerMs);
|
|
32
|
+
this.lastRefill = now;
|
|
33
|
+
if (this.tokens >= 1) {
|
|
34
|
+
this.tokens -= 1;
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
// Wait for enough time to refill one token
|
|
38
|
+
const waitMs = Math.ceil((1 - this.tokens) / this.ratePerMs);
|
|
39
|
+
await new Promise(r => setTimeout(r, waitMs));
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
exports.TokenBucket = TokenBucket;
|
|
44
|
+
// ─── Concurrency Limiter ───────────────────────────────────────────────────────
|
|
45
|
+
class ConcurrencyLimiter {
|
|
46
|
+
constructor(max) {
|
|
47
|
+
this.max = max;
|
|
48
|
+
this.running = 0;
|
|
49
|
+
this.queue = [];
|
|
50
|
+
}
|
|
51
|
+
async acquire() {
|
|
52
|
+
if (this.running < this.max) {
|
|
53
|
+
this.running++;
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
await new Promise(resolve => this.queue.push(resolve));
|
|
57
|
+
this.running++;
|
|
58
|
+
}
|
|
59
|
+
release() {
|
|
60
|
+
this.running--;
|
|
61
|
+
if (this.queue.length > 0) {
|
|
62
|
+
const next = this.queue.shift();
|
|
63
|
+
next();
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
exports.ConcurrencyLimiter = ConcurrencyLimiter;
|
|
68
|
+
// ─── Singleton shared across all adapter instances ─────────────────────────────
|
|
69
|
+
const _bucket = new TokenBucket(exports.RATE_LIMIT_RPS, 10);
|
|
70
|
+
const _concurrency = new ConcurrencyLimiter(exports.MAX_CONCURRENCY);
|
|
71
|
+
// ─── Error classification ──────────────────────────────────────────────────────
|
|
72
|
+
function isRetryableError(err) {
|
|
9
73
|
const code = err?.code || err?.status || err?.response?.status;
|
|
10
74
|
if (code === 429)
|
|
11
75
|
return true;
|
|
12
|
-
if (code === 500 || code === 503)
|
|
76
|
+
if (code === 500 || code === 503 || code === 502)
|
|
13
77
|
return true;
|
|
14
78
|
if (code === 403) {
|
|
15
79
|
const reason = err?.errors?.[0]?.reason || '';
|
|
16
|
-
if (reason === 'rateLimitExceeded')
|
|
80
|
+
if (reason === 'rateLimitExceeded' || reason === 'userRateLimitExceeded')
|
|
17
81
|
return true;
|
|
18
82
|
if (typeof err?.body === 'string' && err.body.includes('rateLimitExceeded'))
|
|
19
83
|
return true;
|
|
@@ -22,18 +86,30 @@ function isRateLimitError(err) {
|
|
|
22
86
|
}
|
|
23
87
|
return false;
|
|
24
88
|
}
|
|
89
|
+
function getRetryDelay(err, attempt) {
|
|
90
|
+
// Honour server-provided Retry-After
|
|
91
|
+
if (err?.retryAfter)
|
|
92
|
+
return err.retryAfter * 1000;
|
|
93
|
+
// Full jitter exponential backoff: random in [0, min(cap, base * 2^attempt)]
|
|
94
|
+
const cap = Math.min(exports.MAX_DELAY, exports.BASE_DELAY * Math.pow(2, attempt));
|
|
95
|
+
return Math.random() * cap;
|
|
96
|
+
}
|
|
97
|
+
// ─── Public API ────────────────────────────────────────────────────────────────
|
|
25
98
|
async function withRetry(fn, attempt = 1) {
|
|
99
|
+
await _bucket.acquire();
|
|
100
|
+
await _concurrency.acquire();
|
|
26
101
|
try {
|
|
27
102
|
return await fn();
|
|
28
103
|
}
|
|
29
104
|
catch (err) {
|
|
30
|
-
if (
|
|
31
|
-
const delay = err
|
|
32
|
-
? err.retryAfter * 1000
|
|
33
|
-
: exports.BASE_DELAY * Math.pow(2, attempt - 1) + Math.random() * 500;
|
|
105
|
+
if (isRetryableError(err) && attempt <= exports.MAX_RETRIES) {
|
|
106
|
+
const delay = getRetryDelay(err, attempt);
|
|
34
107
|
await new Promise(r => setTimeout(r, delay));
|
|
35
108
|
return withRetry(fn, attempt + 1);
|
|
36
109
|
}
|
|
37
110
|
throw err;
|
|
38
111
|
}
|
|
112
|
+
finally {
|
|
113
|
+
_concurrency.release();
|
|
114
|
+
}
|
|
39
115
|
}
|
|
@@ -1,10 +1,16 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.SheetsTableClient = void 0;
|
|
4
|
-
const
|
|
4
|
+
const uuid_1 = require("../base/uuid");
|
|
5
5
|
const metadata_1 = require("../base/metadata");
|
|
6
6
|
const helpers_1 = require("./helpers");
|
|
7
7
|
const retry_1 = require("./retry");
|
|
8
|
+
// ─── Utility ──────────────────────────────────────────────────────────────────
|
|
9
|
+
function resolveIdField(fields) {
|
|
10
|
+
if (Object.prototype.hasOwnProperty.call(fields, 'id'))
|
|
11
|
+
return 'id';
|
|
12
|
+
return Object.keys(fields).find(name => name.endsWith('_id') || name.endsWith('Id') || name.toLowerCase() === 'id');
|
|
13
|
+
}
|
|
8
14
|
// ─── Table Client ─────────────────────────────────────────────────────────────
|
|
9
15
|
class SheetsTableClient {
|
|
10
16
|
constructor(adapter, modelName, sheetMapping) {
|
|
@@ -146,20 +152,20 @@ class SheetsTableClient {
|
|
|
146
152
|
const rows = await this.findMany({ where: args?.where });
|
|
147
153
|
return rows.length;
|
|
148
154
|
}
|
|
155
|
+
assignId(data, fields) {
|
|
156
|
+
const idFieldName = resolveIdField(fields);
|
|
157
|
+
if (!idFieldName)
|
|
158
|
+
return;
|
|
159
|
+
const fieldDef = fields[idFieldName];
|
|
160
|
+
const rawType = typeof fieldDef === 'string' ? fieldDef : (fieldDef?.ts || fieldDef?.sql || '');
|
|
161
|
+
const isStringType = ['string', 'uuid', 'uniqueidentifier', 'nvarchar', 'varchar', 'text'].includes(rawType.toLowerCase());
|
|
162
|
+
if (isStringType && !data[idFieldName]) {
|
|
163
|
+
data[idFieldName] = (0, uuid_1.generateUUID)();
|
|
164
|
+
}
|
|
165
|
+
}
|
|
149
166
|
async create(args) {
|
|
150
|
-
const fields = this.fields;
|
|
151
|
-
const idFieldName = Object.prototype.hasOwnProperty.call(fields, 'id')
|
|
152
|
-
? 'id'
|
|
153
|
-
: Object.keys(fields).find(name => name.endsWith('_id') || name.endsWith('Id') || name.toLowerCase() === 'id');
|
|
154
167
|
const data = { ...args.data };
|
|
155
|
-
|
|
156
|
-
const fieldDef = fields[idFieldName];
|
|
157
|
-
const rawType = typeof fieldDef === 'string' ? fieldDef : (fieldDef?.ts || fieldDef?.sql || '');
|
|
158
|
-
const isStringType = ['string', 'uuid', 'uniqueidentifier', 'nvarchar', 'varchar', 'text'].includes(rawType.toLowerCase());
|
|
159
|
-
if (isStringType && !data[idFieldName]) {
|
|
160
|
-
data[idFieldName] = (0, crypto_1.randomUUID)();
|
|
161
|
-
}
|
|
162
|
-
}
|
|
168
|
+
this.assignId(data, this.fields);
|
|
163
169
|
const cols = Object.keys(data).filter(k => data[k] !== undefined);
|
|
164
170
|
const headers = await this.getOrCreateHeaders(cols);
|
|
165
171
|
for (const h of headers) {
|
|
@@ -174,28 +180,16 @@ class SheetsTableClient {
|
|
|
174
180
|
async createMany(args) {
|
|
175
181
|
if (args.data.length === 0)
|
|
176
182
|
return { count: 0 };
|
|
183
|
+
const fields = this.fields;
|
|
177
184
|
const allKeys = new Set();
|
|
178
|
-
for (const row of args.data)
|
|
185
|
+
for (const row of args.data)
|
|
179
186
|
Object.keys(row).forEach(k => allKeys.add(k));
|
|
180
|
-
}
|
|
181
187
|
const cols = [...allKeys];
|
|
182
|
-
const
|
|
183
|
-
const idFieldName = Object.prototype.hasOwnProperty.call(fields, 'id')
|
|
184
|
-
? 'id'
|
|
185
|
-
: Object.keys(fields).find(name => name.endsWith('_id') || name.endsWith('Id') || name.toLowerCase() === 'id');
|
|
186
|
-
const prepared = [];
|
|
187
|
-
for (const item of args.data) {
|
|
188
|
+
const prepared = args.data.map(item => {
|
|
188
189
|
const data = { ...item };
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
const isStringType = ['string', 'uuid', 'uniqueidentifier', 'nvarchar', 'varchar', 'text'].includes(rawType.toLowerCase());
|
|
193
|
-
if (isStringType && !data[idFieldName]) {
|
|
194
|
-
data[idFieldName] = (0, crypto_1.randomUUID)();
|
|
195
|
-
}
|
|
196
|
-
}
|
|
197
|
-
prepared.push(data);
|
|
198
|
-
}
|
|
190
|
+
this.assignId(data, fields);
|
|
191
|
+
return data;
|
|
192
|
+
});
|
|
199
193
|
const headers = await this.getOrCreateHeaders(cols);
|
|
200
194
|
const batchValues = prepared.map(data => {
|
|
201
195
|
for (const h of headers) {
|
|
@@ -238,16 +232,26 @@ class SheetsTableClient {
|
|
|
238
232
|
async updateMany(args) {
|
|
239
233
|
const { headers, rows } = await this.readAllRows();
|
|
240
234
|
const matching = rows.filter(r => (0, helpers_1.matchWhere)(r, args.where));
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
235
|
+
if (matching.length === 0)
|
|
236
|
+
return { count: 0 };
|
|
237
|
+
// Batch all writes into a single batchUpdate call
|
|
238
|
+
const api = await this.adapter.getSheets();
|
|
239
|
+
const spreadsheetId = this.adapter.config.spreadsheetId;
|
|
240
|
+
const valueRanges = matching.map(target => {
|
|
241
|
+
const updated = { ...target, ...args.data };
|
|
242
|
+
return {
|
|
243
|
+
range: `${this.escSheetName}!A${target.__row}`,
|
|
244
|
+
values: [this.rowToValues(updated, headers)],
|
|
245
|
+
};
|
|
246
|
+
});
|
|
247
|
+
await (0, retry_1.withRetry)(() => api.spreadsheets.values.batchUpdate({
|
|
248
|
+
spreadsheetId,
|
|
249
|
+
requestBody: { valueInputOption: 'RAW', data: valueRanges },
|
|
250
|
+
}));
|
|
247
251
|
return { count: matching.length };
|
|
248
252
|
}
|
|
249
253
|
async delete(args) {
|
|
250
|
-
const {
|
|
254
|
+
const { rows } = await this.readAllRows();
|
|
251
255
|
const matching = rows.filter(r => (0, helpers_1.matchWhere)(r, args.where));
|
|
252
256
|
if (matching.length === 0)
|
|
253
257
|
throw new Error('No record found matching where clause');
|
|
@@ -260,17 +264,30 @@ class SheetsTableClient {
|
|
|
260
264
|
return rest;
|
|
261
265
|
}
|
|
262
266
|
async deleteMany(args) {
|
|
263
|
-
const {
|
|
267
|
+
const { rows } = await this.readAllRows();
|
|
264
268
|
const matching = rows.filter(r => (0, helpers_1.matchWhere)(r, args?.where));
|
|
265
269
|
if (matching.length === 0)
|
|
266
270
|
return { count: 0 };
|
|
267
271
|
const meta = await this.adapter.getSheetMeta(this.sheetName);
|
|
268
272
|
if (!meta)
|
|
269
273
|
throw new Error(`Sheet "${this.sheetName}" not found`);
|
|
274
|
+
// Batch all deletes into a single batchUpdate (sorted DESC to keep row indices stable)
|
|
275
|
+
const api = await this.adapter.getSheets();
|
|
270
276
|
const sorted = [...matching].sort((a, b) => b.__row - a.__row);
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
277
|
+
const requests = sorted.map(target => ({
|
|
278
|
+
deleteDimension: {
|
|
279
|
+
range: {
|
|
280
|
+
sheetId: meta.sheetId,
|
|
281
|
+
dimension: 'ROWS',
|
|
282
|
+
startIndex: target.__row - 1,
|
|
283
|
+
endIndex: target.__row,
|
|
284
|
+
},
|
|
285
|
+
},
|
|
286
|
+
}));
|
|
287
|
+
await (0, retry_1.withRetry)(() => api.spreadsheets.batchUpdate({
|
|
288
|
+
spreadsheetId: this.adapter.config.spreadsheetId,
|
|
289
|
+
requestBody: { requests },
|
|
290
|
+
}));
|
|
274
291
|
return { count: matching.length };
|
|
275
292
|
}
|
|
276
293
|
async deleteAll() {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { generateUUID } from './base/uuid';
|
|
2
2
|
|
|
3
3
|
import {
|
|
4
4
|
An5SheetsAdapter,
|
|
@@ -317,7 +317,7 @@ export class AdapterTableClient<T = any> {
|
|
|
317
317
|
const rawType = typeof fieldDef === 'string' ? fieldDef : (fieldDef?.ts || fieldDef?.sql || fieldDef?.type || '');
|
|
318
318
|
const normalizedType = String(rawType).toLowerCase();
|
|
319
319
|
const isStringType = ['string', 'uuid', 'uniqueidentifier', 'nvarchar', 'varchar', 'text'].includes(normalizedType);
|
|
320
|
-
if (isStringType && !data[idFieldName]) data[idFieldName] =
|
|
320
|
+
if (isStringType && !data[idFieldName]) data[idFieldName] = generateUUID();
|
|
321
321
|
}
|
|
322
322
|
|
|
323
323
|
const cols = Object.keys(data).filter(k => data[k] !== undefined);
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cross-platform UUID generator safe for both Browser and Node.js environments.
|
|
3
|
+
* Uses crypto.randomUUID() when available, with a standard RFC4122 v4 fallback.
|
|
4
|
+
*/
|
|
5
|
+
export function generateUUID(): string {
|
|
6
|
+
if (typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function') {
|
|
7
|
+
return crypto.randomUUID();
|
|
8
|
+
}
|
|
9
|
+
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
|
|
10
|
+
const r = (Math.random() * 16) | 0;
|
|
11
|
+
const v = c === 'x' ? r : (r & 0x3) | 0x8;
|
|
12
|
+
return v.toString(16);
|
|
13
|
+
});
|
|
14
|
+
}
|
|
@@ -5,14 +5,21 @@ import { SheetsTableClient } from './tableClient';
|
|
|
5
5
|
import { SheetMeta } from './types';
|
|
6
6
|
import { withRetry } from './retry';
|
|
7
7
|
|
|
8
|
-
// ─── Fetch-based API proxy for OAuth Access Token (browser-compatible)
|
|
8
|
+
// ─── Fetch-based API proxy for OAuth Access Token / API Key (browser-compatible) ──
|
|
9
9
|
|
|
10
|
-
function createFetchApi(spreadsheetId: string, accessToken
|
|
10
|
+
function createFetchApi(spreadsheetId: string, accessToken?: string, apiKey?: string) {
|
|
11
11
|
const base = `https://sheets.googleapis.com/v4/spreadsheets/${spreadsheetId}`;
|
|
12
|
-
const headers
|
|
12
|
+
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
|
|
13
|
+
if (accessToken) {
|
|
14
|
+
headers['Authorization'] = `Bearer ${accessToken}`;
|
|
15
|
+
}
|
|
13
16
|
|
|
14
17
|
async function request<T>(method: string, path: string, body?: any): Promise<T> {
|
|
15
|
-
|
|
18
|
+
let url = path.startsWith('http') ? path : `${base}${path}`;
|
|
19
|
+
if (apiKey) {
|
|
20
|
+
const sep = url.includes('?') ? '&' : '?';
|
|
21
|
+
url += `${sep}key=${encodeURIComponent(apiKey)}`;
|
|
22
|
+
}
|
|
16
23
|
const opts: RequestInit = { method, headers };
|
|
17
24
|
if (body) opts.body = JSON.stringify(body);
|
|
18
25
|
const res = await fetch(url, opts);
|
|
@@ -79,17 +86,27 @@ export class An5SheetsAdapter {
|
|
|
79
86
|
this.config = resolveConfig(config);
|
|
80
87
|
}
|
|
81
88
|
|
|
82
|
-
private get
|
|
83
|
-
return !!(this.config as any).accessToken;
|
|
89
|
+
private get isFetchMode(): boolean {
|
|
90
|
+
return !!(this.config as any).accessToken || !!(this.config as any).apiKey;
|
|
84
91
|
}
|
|
85
92
|
|
|
86
93
|
async getSheets(): Promise<any> {
|
|
87
|
-
if (this.
|
|
94
|
+
if (this.isFetchMode) {
|
|
88
95
|
if (!this.fetchApi) {
|
|
89
|
-
this.fetchApi = createFetchApi(
|
|
96
|
+
this.fetchApi = createFetchApi(
|
|
97
|
+
this.config.spreadsheetId,
|
|
98
|
+
(this.config as any).accessToken,
|
|
99
|
+
(this.config as any).apiKey
|
|
100
|
+
);
|
|
90
101
|
}
|
|
91
102
|
return this.fetchApi;
|
|
92
103
|
}
|
|
104
|
+
const isBrowser = typeof window !== 'undefined' && typeof window.document !== 'undefined';
|
|
105
|
+
if (isBrowser) {
|
|
106
|
+
throw new Error(
|
|
107
|
+
'Google Sheets adapter in browser mode requires "accessToken" or "apiKey" configuration.'
|
|
108
|
+
);
|
|
109
|
+
}
|
|
93
110
|
if (!this.sheets) {
|
|
94
111
|
const { google } = await import('googleapis');
|
|
95
112
|
const auth = new google.auth.JWT({
|
|
@@ -10,6 +10,8 @@ export interface An5SheetsAdapterConfig {
|
|
|
10
10
|
* When provided, the adapter uses raw fetch() instead of googleapis JWT, making it compatible
|
|
11
11
|
* with browser environments. Mutually exclusive with clientEmail/privateKey/credentials. */
|
|
12
12
|
accessToken?: string;
|
|
13
|
+
/** Google API key for browser-based API access. */
|
|
14
|
+
apiKey?: string;
|
|
13
15
|
}
|
|
14
16
|
|
|
15
17
|
export function normalizeKey(key: string): string {
|
|
@@ -20,10 +22,11 @@ export function normalizeKey(key: string): string {
|
|
|
20
22
|
}
|
|
21
23
|
|
|
22
24
|
export function resolveConfig(config: An5SheetsAdapterConfig) {
|
|
23
|
-
if (config.accessToken) {
|
|
25
|
+
if (config.accessToken || config.apiKey) {
|
|
24
26
|
return {
|
|
25
27
|
spreadsheetId: config.spreadsheetId,
|
|
26
28
|
accessToken: config.accessToken,
|
|
29
|
+
apiKey: config.apiKey,
|
|
27
30
|
sheetMapping: config.sheetMapping,
|
|
28
31
|
};
|
|
29
32
|
}
|
|
@@ -40,6 +40,11 @@ export function matchWhere(row: Record<string, any>, where: any): boolean {
|
|
|
40
40
|
if (!value.every((v: any) => matchWhere(row, v))) return false;
|
|
41
41
|
continue;
|
|
42
42
|
}
|
|
43
|
+
if (key === 'NOT') {
|
|
44
|
+
const notClauses = Array.isArray(value) ? value : [value];
|
|
45
|
+
if (notClauses.some((v: any) => matchWhere(row, v))) return false;
|
|
46
|
+
continue;
|
|
47
|
+
}
|
|
43
48
|
|
|
44
49
|
const cellVal = row[key];
|
|
45
50
|
|
|
@@ -47,28 +52,42 @@ export function matchWhere(row: Record<string, any>, where: any): boolean {
|
|
|
47
52
|
if (cellVal !== null && cellVal !== undefined) return false;
|
|
48
53
|
} else if (typeof value === 'object' && !(value instanceof Date)) {
|
|
49
54
|
const v = value as any;
|
|
55
|
+
const caseInsensitive = v.mode === 'insensitive';
|
|
56
|
+
const normalize = (s: any) => caseInsensitive ? String(s ?? '').toLowerCase() : s;
|
|
57
|
+
|
|
50
58
|
if (v.not !== undefined) {
|
|
51
59
|
if (v.not === null) { if (cellVal === null || cellVal === undefined) return false; }
|
|
52
|
-
else if (cellVal == v.not) return false;
|
|
60
|
+
else if (normalize(cellVal) == normalize(v.not)) return false;
|
|
61
|
+
}
|
|
62
|
+
if (v.equals !== undefined && normalize(cellVal) != normalize(v.equals)) return false;
|
|
63
|
+
if (v.contains !== undefined) {
|
|
64
|
+
if (!cellVal || !normalize(String(cellVal)).includes(normalize(v.contains))) return false;
|
|
65
|
+
}
|
|
66
|
+
if (v.startsWith !== undefined) {
|
|
67
|
+
if (!cellVal || !normalize(String(cellVal)).startsWith(normalize(v.startsWith))) return false;
|
|
68
|
+
}
|
|
69
|
+
if (v.endsWith !== undefined) {
|
|
70
|
+
if (!cellVal || !normalize(String(cellVal)).endsWith(normalize(v.endsWith))) return false;
|
|
53
71
|
}
|
|
54
|
-
if (v.equals !== undefined && cellVal != v.equals) return false;
|
|
55
|
-
if (v.contains !== undefined && (!cellVal || !String(cellVal).includes(v.contains))) return false;
|
|
56
|
-
if (v.startsWith !== undefined && (!cellVal || !String(cellVal).startsWith(v.startsWith))) return false;
|
|
57
|
-
if (v.endsWith !== undefined && (!cellVal || !String(cellVal).endsWith(v.endsWith))) return false;
|
|
58
72
|
if (v.gte !== undefined && !(Number(cellVal) >= Number(v.gte))) return false;
|
|
59
73
|
if (v.lte !== undefined && !(Number(cellVal) <= Number(v.lte))) return false;
|
|
60
74
|
if (v.gt !== undefined && !(Number(cellVal) > Number(v.gt))) return false;
|
|
61
75
|
if (v.lt !== undefined && !(Number(cellVal) < Number(v.lt))) return false;
|
|
62
76
|
if (v.in !== undefined) {
|
|
63
|
-
if (!Array.isArray(v.in) || !v.in.
|
|
77
|
+
if (!Array.isArray(v.in) || !v.in.some((item: any) => item == cellVal)) return false;
|
|
78
|
+
}
|
|
79
|
+
if (v.notIn !== undefined) {
|
|
80
|
+
if (Array.isArray(v.notIn) && v.notIn.some((item: any) => item == cellVal)) return false;
|
|
64
81
|
}
|
|
65
82
|
} else {
|
|
83
|
+
// loose equality to handle string/number coercion from sheet cells
|
|
66
84
|
if (cellVal != value) return false;
|
|
67
85
|
}
|
|
68
86
|
}
|
|
69
87
|
return true;
|
|
70
88
|
}
|
|
71
89
|
|
|
90
|
+
|
|
72
91
|
export function buildOrderBy(orderBy: any): string {
|
|
73
92
|
if (!orderBy) return '';
|
|
74
93
|
const entries = Array.isArray(orderBy) ? orderBy : [orderBy];
|
|
@@ -30,6 +30,8 @@ export function parseSheetsConnectionString(url: string): An5SheetsAdapterConfig
|
|
|
30
30
|
|
|
31
31
|
if (config.clientemail) result.clientEmail = config.clientemail;
|
|
32
32
|
if (config.privatekey) result.privateKey = config.privatekey;
|
|
33
|
+
if (config.accesstoken) result.accessToken = config.accesstoken;
|
|
34
|
+
if (config.apikey) result.apiKey = config.apikey;
|
|
33
35
|
|
|
34
36
|
if (config.sheetmapping) {
|
|
35
37
|
const mapping: Record<string, string> = {};
|
|
@@ -1,32 +1,126 @@
|
|
|
1
|
-
// ─── Retry helper
|
|
1
|
+
// ─── Rate Limit & Retry helper ─────────────────────────────────────────────────
|
|
2
|
+
//
|
|
3
|
+
// Google Sheets API quota (default): 100 requests per 100 seconds per project.
|
|
4
|
+
// Strategy:
|
|
5
|
+
// 1. Token-bucket rate limiter — proactively throttles before hitting quota
|
|
6
|
+
// 2. Concurrency limiter — max N parallel requests to avoid burst
|
|
7
|
+
// 3. Exponential backoff + jitter — reactive recovery when 429/403/5xx hit
|
|
2
8
|
|
|
3
|
-
|
|
4
|
-
export const BASE_DELAY = 1000;
|
|
9
|
+
// ─── Config ────────────────────────────────────────────────────────────────────
|
|
5
10
|
|
|
6
|
-
|
|
11
|
+
export const MAX_RETRIES = 5;
|
|
12
|
+
export const BASE_DELAY = 1000; // ms
|
|
13
|
+
export const MAX_DELAY = 32_000; // ms cap for backoff
|
|
14
|
+
export const MAX_CONCURRENCY = 4; // parallel in-flight requests
|
|
15
|
+
export const RATE_LIMIT_RPS = 0.9; // tokens/second (stay under 1 req/s safely)
|
|
16
|
+
|
|
17
|
+
// ─── Token Bucket ──────────────────────────────────────────────────────────────
|
|
18
|
+
|
|
19
|
+
class TokenBucket {
|
|
20
|
+
private tokens: number;
|
|
21
|
+
private lastRefill: number;
|
|
22
|
+
private readonly ratePerMs: number;
|
|
23
|
+
private readonly capacity: number;
|
|
24
|
+
|
|
25
|
+
constructor(ratePerSecond: number, capacity: number) {
|
|
26
|
+
this.ratePerMs = ratePerSecond / 1000;
|
|
27
|
+
this.capacity = capacity;
|
|
28
|
+
this.tokens = capacity;
|
|
29
|
+
this.lastRefill = Date.now();
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** Wait until a token is available, then consume one. */
|
|
33
|
+
async acquire(): Promise<void> {
|
|
34
|
+
while (true) {
|
|
35
|
+
const now = Date.now();
|
|
36
|
+
const elapsed = now - this.lastRefill;
|
|
37
|
+
this.tokens = Math.min(this.capacity, this.tokens + elapsed * this.ratePerMs);
|
|
38
|
+
this.lastRefill = now;
|
|
39
|
+
|
|
40
|
+
if (this.tokens >= 1) {
|
|
41
|
+
this.tokens -= 1;
|
|
42
|
+
return;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// Wait for enough time to refill one token
|
|
46
|
+
const waitMs = Math.ceil((1 - this.tokens) / this.ratePerMs);
|
|
47
|
+
await new Promise(r => setTimeout(r, waitMs));
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// ─── Concurrency Limiter ───────────────────────────────────────────────────────
|
|
53
|
+
|
|
54
|
+
class ConcurrencyLimiter {
|
|
55
|
+
private running = 0;
|
|
56
|
+
private queue: Array<() => void> = [];
|
|
57
|
+
|
|
58
|
+
constructor(private readonly max: number) {}
|
|
59
|
+
|
|
60
|
+
async acquire(): Promise<void> {
|
|
61
|
+
if (this.running < this.max) {
|
|
62
|
+
this.running++;
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
65
|
+
await new Promise<void>(resolve => this.queue.push(resolve));
|
|
66
|
+
this.running++;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
release(): void {
|
|
70
|
+
this.running--;
|
|
71
|
+
if (this.queue.length > 0) {
|
|
72
|
+
const next = this.queue.shift()!;
|
|
73
|
+
next();
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// ─── Singleton shared across all adapter instances ─────────────────────────────
|
|
79
|
+
|
|
80
|
+
const _bucket = new TokenBucket(RATE_LIMIT_RPS, 10);
|
|
81
|
+
const _concurrency = new ConcurrencyLimiter(MAX_CONCURRENCY);
|
|
82
|
+
|
|
83
|
+
// ─── Error classification ──────────────────────────────────────────────────────
|
|
84
|
+
|
|
85
|
+
function isRetryableError(err: any): boolean {
|
|
7
86
|
const code = err?.code || err?.status || err?.response?.status;
|
|
8
87
|
if (code === 429) return true;
|
|
9
|
-
if (code === 500 || code === 503) return true;
|
|
88
|
+
if (code === 500 || code === 503 || code === 502) return true;
|
|
10
89
|
if (code === 403) {
|
|
11
90
|
const reason = err?.errors?.[0]?.reason || '';
|
|
12
|
-
if (reason === 'rateLimitExceeded') return true;
|
|
91
|
+
if (reason === 'rateLimitExceeded' || reason === 'userRateLimitExceeded') return true;
|
|
13
92
|
if (typeof err?.body === 'string' && err.body.includes('rateLimitExceeded')) return true;
|
|
14
93
|
if (typeof err?.message === 'string' && err.message.includes('rateLimitExceeded')) return true;
|
|
15
94
|
}
|
|
16
95
|
return false;
|
|
17
96
|
}
|
|
18
97
|
|
|
98
|
+
function getRetryDelay(err: any, attempt: number): number {
|
|
99
|
+
// Honour server-provided Retry-After
|
|
100
|
+
if (err?.retryAfter) return err.retryAfter * 1000;
|
|
101
|
+
// Full jitter exponential backoff: random in [0, min(cap, base * 2^attempt)]
|
|
102
|
+
const cap = Math.min(MAX_DELAY, BASE_DELAY * Math.pow(2, attempt));
|
|
103
|
+
return Math.random() * cap;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// ─── Public API ────────────────────────────────────────────────────────────────
|
|
107
|
+
|
|
19
108
|
export async function withRetry<T>(fn: () => Promise<T>, attempt = 1): Promise<T> {
|
|
109
|
+
await _bucket.acquire();
|
|
110
|
+
await _concurrency.acquire();
|
|
20
111
|
try {
|
|
21
112
|
return await fn();
|
|
22
113
|
} catch (err: any) {
|
|
23
|
-
if (
|
|
24
|
-
const delay = err
|
|
25
|
-
? err.retryAfter * 1000
|
|
26
|
-
: BASE_DELAY * Math.pow(2, attempt - 1) + Math.random() * 500;
|
|
114
|
+
if (isRetryableError(err) && attempt <= MAX_RETRIES) {
|
|
115
|
+
const delay = getRetryDelay(err, attempt);
|
|
27
116
|
await new Promise(r => setTimeout(r, delay));
|
|
28
117
|
return withRetry(fn, attempt + 1);
|
|
29
118
|
}
|
|
30
119
|
throw err;
|
|
120
|
+
} finally {
|
|
121
|
+
_concurrency.release();
|
|
31
122
|
}
|
|
32
123
|
}
|
|
124
|
+
|
|
125
|
+
/** Expose for testing / custom configurations */
|
|
126
|
+
export { TokenBucket, ConcurrencyLimiter };
|
|
@@ -1,9 +1,18 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { generateUUID } from '../base/uuid';
|
|
2
2
|
import { getFieldsForModel } from '../base/metadata';
|
|
3
3
|
import type { An5SheetsAdapter } from './adapter';
|
|
4
4
|
import { buildOrderBy, coerceCell, esc, matchWhere, resolveSheetName, sortRows } from './helpers';
|
|
5
5
|
import { withRetry } from './retry';
|
|
6
6
|
|
|
7
|
+
// ─── Utility ──────────────────────────────────────────────────────────────────
|
|
8
|
+
|
|
9
|
+
function resolveIdField(fields: Record<string, any>): string | undefined {
|
|
10
|
+
if (Object.prototype.hasOwnProperty.call(fields, 'id')) return 'id';
|
|
11
|
+
return Object.keys(fields).find(
|
|
12
|
+
name => name.endsWith('_id') || name.endsWith('Id') || name.toLowerCase() === 'id'
|
|
13
|
+
);
|
|
14
|
+
}
|
|
15
|
+
|
|
7
16
|
// ─── Table Client ─────────────────────────────────────────────────────────────
|
|
8
17
|
|
|
9
18
|
export class SheetsTableClient<T = any> {
|
|
@@ -167,21 +176,20 @@ export class SheetsTableClient<T = any> {
|
|
|
167
176
|
return rows.length;
|
|
168
177
|
}
|
|
169
178
|
|
|
170
|
-
|
|
171
|
-
const
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
179
|
+
private assignId(data: any, fields: Record<string, any>): void {
|
|
180
|
+
const idFieldName = resolveIdField(fields);
|
|
181
|
+
if (!idFieldName) return;
|
|
182
|
+
const fieldDef = fields[idFieldName];
|
|
183
|
+
const rawType = typeof fieldDef === 'string' ? fieldDef : (fieldDef?.ts || fieldDef?.sql || '');
|
|
184
|
+
const isStringType = ['string', 'uuid', 'uniqueidentifier', 'nvarchar', 'varchar', 'text'].includes(rawType.toLowerCase());
|
|
185
|
+
if (isStringType && !data[idFieldName]) {
|
|
186
|
+
data[idFieldName] = generateUUID();
|
|
187
|
+
}
|
|
188
|
+
}
|
|
175
189
|
|
|
190
|
+
async create(args: { data: Partial<T> }): Promise<T> {
|
|
176
191
|
const data: any = { ...args.data } as any;
|
|
177
|
-
|
|
178
|
-
const fieldDef = fields[idFieldName];
|
|
179
|
-
const rawType = typeof fieldDef === 'string' ? fieldDef : (fieldDef?.ts || fieldDef?.sql || '');
|
|
180
|
-
const isStringType = ['string', 'uuid', 'uniqueidentifier', 'nvarchar', 'varchar', 'text'].includes(rawType.toLowerCase());
|
|
181
|
-
if (isStringType && !data[idFieldName]) {
|
|
182
|
-
data[idFieldName] = randomUUID();
|
|
183
|
-
}
|
|
184
|
-
}
|
|
192
|
+
this.assignId(data, this.fields);
|
|
185
193
|
|
|
186
194
|
const cols = Object.keys(data).filter(k => data[k] !== undefined);
|
|
187
195
|
const headers = await this.getOrCreateHeaders(cols);
|
|
@@ -200,36 +208,20 @@ export class SheetsTableClient<T = any> {
|
|
|
200
208
|
async createMany(args: { data: Partial<T>[]; skipDuplicates?: boolean }): Promise<{ count: number }> {
|
|
201
209
|
if (args.data.length === 0) return { count: 0 };
|
|
202
210
|
|
|
211
|
+
const fields = this.fields;
|
|
203
212
|
const allKeys = new Set<string>();
|
|
204
|
-
for (const row of args.data)
|
|
205
|
-
Object.keys(row as any).forEach(k => allKeys.add(k));
|
|
206
|
-
}
|
|
213
|
+
for (const row of args.data) Object.keys(row as any).forEach(k => allKeys.add(k));
|
|
207
214
|
const cols = [...allKeys];
|
|
208
|
-
const fields = this.fields;
|
|
209
|
-
const idFieldName = Object.prototype.hasOwnProperty.call(fields, 'id')
|
|
210
|
-
? 'id'
|
|
211
|
-
: Object.keys(fields).find(name => name.endsWith('_id') || name.endsWith('Id') || name.toLowerCase() === 'id');
|
|
212
215
|
|
|
213
|
-
const prepared: Record<string, any>[] =
|
|
214
|
-
for (const item of args.data) {
|
|
216
|
+
const prepared: Record<string, any>[] = args.data.map(item => {
|
|
215
217
|
const data: any = { ...item } as any;
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
const isStringType = ['string', 'uuid', 'uniqueidentifier', 'nvarchar', 'varchar', 'text'].includes(rawType.toLowerCase());
|
|
220
|
-
if (isStringType && !data[idFieldName]) {
|
|
221
|
-
data[idFieldName] = randomUUID();
|
|
222
|
-
}
|
|
223
|
-
}
|
|
224
|
-
prepared.push(data);
|
|
225
|
-
}
|
|
218
|
+
this.assignId(data, fields);
|
|
219
|
+
return data;
|
|
220
|
+
});
|
|
226
221
|
|
|
227
222
|
const headers = await this.getOrCreateHeaders(cols);
|
|
228
|
-
|
|
229
223
|
const batchValues = prepared.map(data => {
|
|
230
|
-
for (const h of headers) {
|
|
231
|
-
if (data[h] === undefined) data[h] = null;
|
|
232
|
-
}
|
|
224
|
+
for (const h of headers) { if (data[h] === undefined) data[h] = null; }
|
|
233
225
|
return this.rowToValues(data, headers);
|
|
234
226
|
});
|
|
235
227
|
|
|
@@ -264,18 +256,30 @@ export class SheetsTableClient<T = any> {
|
|
|
264
256
|
async updateMany(args: { where?: any; data: Partial<T> }): Promise<{ count: number }> {
|
|
265
257
|
const { headers, rows } = await this.readAllRows();
|
|
266
258
|
const matching = rows.filter(r => matchWhere(r, args.where));
|
|
259
|
+
if (matching.length === 0) return { count: 0 };
|
|
267
260
|
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
261
|
+
// Batch all writes into a single batchUpdate call
|
|
262
|
+
const api = await this.adapter.getSheets();
|
|
263
|
+
const spreadsheetId = this.adapter.config.spreadsheetId;
|
|
264
|
+
const valueRanges = matching.map(target => {
|
|
265
|
+
const updated = { ...target, ...args.data as any };
|
|
266
|
+
return {
|
|
267
|
+
range: `${this.escSheetName}!A${target.__row}`,
|
|
268
|
+
values: [this.rowToValues(updated, headers)],
|
|
269
|
+
};
|
|
270
|
+
});
|
|
271
|
+
|
|
272
|
+
await withRetry(() =>
|
|
273
|
+
(api as any).spreadsheets.values.batchUpdate({
|
|
274
|
+
spreadsheetId,
|
|
275
|
+
requestBody: { valueInputOption: 'RAW', data: valueRanges },
|
|
276
|
+
})
|
|
277
|
+
);
|
|
274
278
|
return { count: matching.length };
|
|
275
279
|
}
|
|
276
280
|
|
|
277
281
|
async delete(args: { where: any }): Promise<T> {
|
|
278
|
-
const {
|
|
282
|
+
const { rows } = await this.readAllRows();
|
|
279
283
|
const matching = rows.filter(r => matchWhere(r, args.where));
|
|
280
284
|
if (matching.length === 0) throw new Error('No record found matching where clause');
|
|
281
285
|
|
|
@@ -289,17 +293,33 @@ export class SheetsTableClient<T = any> {
|
|
|
289
293
|
}
|
|
290
294
|
|
|
291
295
|
async deleteMany(args?: { where?: any }): Promise<{ count: number }> {
|
|
292
|
-
const {
|
|
296
|
+
const { rows } = await this.readAllRows();
|
|
293
297
|
const matching = rows.filter(r => matchWhere(r, args?.where));
|
|
294
298
|
if (matching.length === 0) return { count: 0 };
|
|
295
299
|
|
|
296
300
|
const meta = await this.adapter.getSheetMeta(this.sheetName);
|
|
297
301
|
if (!meta) throw new Error(`Sheet "${this.sheetName}" not found`);
|
|
298
302
|
|
|
303
|
+
// Batch all deletes into a single batchUpdate (sorted DESC to keep row indices stable)
|
|
304
|
+
const api = await this.adapter.getSheets();
|
|
299
305
|
const sorted = [...matching].sort((a, b) => b.__row - a.__row);
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
306
|
+
const requests = sorted.map(target => ({
|
|
307
|
+
deleteDimension: {
|
|
308
|
+
range: {
|
|
309
|
+
sheetId: meta.sheetId,
|
|
310
|
+
dimension: 'ROWS',
|
|
311
|
+
startIndex: target.__row - 1,
|
|
312
|
+
endIndex: target.__row,
|
|
313
|
+
},
|
|
314
|
+
},
|
|
315
|
+
}));
|
|
316
|
+
|
|
317
|
+
await withRetry(() =>
|
|
318
|
+
api.spreadsheets.batchUpdate({
|
|
319
|
+
spreadsheetId: this.adapter.config.spreadsheetId,
|
|
320
|
+
requestBody: { requests },
|
|
321
|
+
})
|
|
322
|
+
);
|
|
303
323
|
return { count: matching.length };
|
|
304
324
|
}
|
|
305
325
|
|