@an5/adapters 0.2.5 → 0.2.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/README.md +283 -305
  2. package/dist/an5Adapter.d.ts +142 -7
  3. package/dist/an5Adapter.js +998 -68
  4. package/dist/base/metadata.d.ts +10 -0
  5. package/dist/base/metadata.js +107 -0
  6. package/dist/base/sql.d.ts +19 -1
  7. package/dist/base/sql.js +152 -24
  8. package/dist/base/types.d.ts +10 -1
  9. package/dist/browser.d.ts +4 -1
  10. package/dist/browser.js +9 -2
  11. package/dist/googlesheets/adapter.d.ts +1 -0
  12. package/dist/googlesheets/adapter.js +9 -2
  13. package/dist/googlesheets/config.d.ts +4 -4
  14. package/dist/googlesheets/helpers.js +2 -0
  15. package/dist/googlesheets/parseConnectionString.js +5 -2
  16. package/dist/googlesheets/sqlExecutor.js +29 -24
  17. package/dist/googlesheets/tableClient.js +14 -5
  18. package/dist/index.d.ts +5 -1
  19. package/dist/index.js +15 -1
  20. package/dist/mssql/engine.js +8 -4
  21. package/dist/mysql/engine.js +1 -1
  22. package/dist/nbase/client.d.ts +92 -0
  23. package/dist/nbase/client.js +147 -0
  24. package/dist/nbase/index.d.ts +47 -0
  25. package/dist/nbase/index.js +80 -0
  26. package/dist/postgres/engine.js +1 -1
  27. package/dist/sqlite/browserEngine.d.ts +27 -0
  28. package/dist/sqlite/browserEngine.js +161 -0
  29. package/dist/sqlite/engine.js +15 -3
  30. package/dist/sqlite/index.d.ts +1 -0
  31. package/dist/sqlite/index.js +1 -0
  32. package/dist/typed.d.ts +5 -0
  33. package/dist/typed.js +2 -0
  34. package/dotnet/An5Adapter.cs +1112 -275
  35. package/dotnet/Base/Core.cs +380 -82
  36. package/dotnet/Mssql/MssqlEngine.cs +145 -144
  37. package/dotnet/Postgres/PostgresEngine.cs +149 -121
  38. package/dotnet/Sqlite/SqliteEngine.cs +214 -0
  39. package/golang/an5_adapter.go +314 -0
  40. package/golang/base/base.go +100 -0
  41. package/golang/base/metadata.go +116 -0
  42. package/golang/base/where.go +311 -0
  43. package/golang/go.mod +3 -0
  44. package/golang/mssql/mssql.go +32 -0
  45. package/golang/postgres/postgres.go +37 -0
  46. package/golang/sqlite/sqlite.go +14 -0
  47. package/golang/sqlite_test.go +42 -0
  48. package/golang/table_client.go +1329 -0
  49. package/package.json +26 -9
  50. package/python/an5_adapter.py +184 -97
  51. package/python/base/__init__.py +3 -3
  52. package/python/base/dialects.py +11 -8
  53. package/python/base/metadata.py +59 -10
  54. package/python/base/sql.py +202 -121
  55. package/python/mssql/__init__.py +1 -1
  56. package/python/mssql/provider.py +62 -62
  57. package/python/postgres/__init__.py +1 -1
  58. package/python/postgres/provider.py +15 -15
  59. package/python/sqlite/__init__.py +3 -0
  60. package/python/sqlite/provider.py +83 -0
  61. package/python/table_client.py +636 -216
  62. package/rust/Cargo.toml +31 -0
  63. package/rust/README.md +71 -0
  64. package/rust/src/adapter.rs +1312 -0
  65. package/rust/src/base/dialect.rs +172 -0
  66. package/rust/src/base/metadata.rs +134 -0
  67. package/rust/src/base/mod.rs +17 -0
  68. package/rust/src/base/where_builder.rs +405 -0
  69. package/rust/src/lib.rs +75 -0
  70. package/typescript/src/an5Adapter.ts +1515 -505
  71. package/typescript/src/base/index.ts +3 -3
  72. package/typescript/src/base/metadata.ts +148 -28
  73. package/typescript/src/base/sql.ts +264 -98
  74. package/typescript/src/base/types.ts +33 -24
  75. package/typescript/src/browser.ts +10 -6
  76. package/typescript/src/config.ts +95 -95
  77. package/typescript/src/googlesheets/adapter.ts +269 -259
  78. package/typescript/src/googlesheets/an5SheetsAdapter.ts +3 -3
  79. package/typescript/src/googlesheets/config.ts +47 -47
  80. package/typescript/src/googlesheets/helpers.ts +144 -143
  81. package/typescript/src/googlesheets/index.ts +3 -3
  82. package/typescript/src/googlesheets/parseConnectionString.ts +48 -46
  83. package/typescript/src/googlesheets/retry.ts +126 -126
  84. package/typescript/src/googlesheets/sqlExecutor.ts +304 -301
  85. package/typescript/src/googlesheets/tableClient.ts +461 -455
  86. package/typescript/src/googlesheets/types.ts +6 -6
  87. package/typescript/src/index.ts +48 -27
  88. package/typescript/src/mssql/engine.ts +134 -132
  89. package/typescript/src/mssql/index.ts +1 -1
  90. package/typescript/src/mysql/engine.ts +80 -80
  91. package/typescript/src/mysql/index.ts +1 -1
  92. package/typescript/src/nbase/client.ts +214 -0
  93. package/typescript/src/nbase/index.ts +104 -0
  94. package/typescript/src/postgres/engine.ts +87 -87
  95. package/typescript/src/postgres/index.ts +1 -1
  96. package/typescript/src/sqlite/browserEngine.ts +192 -0
  97. package/typescript/src/sqlite/engine.ts +66 -55
  98. package/typescript/src/sqlite/index.ts +3 -1
  99. package/typescript/src/typed.ts +52 -0
@@ -1,143 +1,144 @@
1
- import { getModelToTable } from '../base/metadata';
2
-
3
- // ─── Helpers ──────────────────────────────────────────────────────────────────
4
-
5
- export function esc(name: string): string {
6
- return name.includes(' ') || /[^\w]/.test(name) ? `'${name.replace(/'/g, "''")}'` : name;
7
- }
8
-
9
- export function resolveSheetName(modelName: string, mapping?: Record<string, string>): string {
10
- if (mapping?.[modelName]) return mapping[modelName];
11
- const modelToTable = getModelToTable();
12
- return modelToTable[modelName] || modelName;
13
- }
14
-
15
- export function matchWhere(row: Record<string, any>, where: any): boolean {
16
- if (!where) return true;
17
-
18
- const cleanWhere: Record<string, any> = {};
19
- for (const [key, value] of Object.entries(where)) {
20
- if (
21
- key.includes('_') &&
22
- value && typeof value === 'object' &&
23
- !(value instanceof Date) &&
24
- !(value as any).in && !(value as any).contains &&
25
- !(value as any).not && !(value as any).gte &&
26
- !(value as any).lte && !(value as any).gt && !(value as any).lt
27
- ) {
28
- Object.assign(cleanWhere, value);
29
- } else {
30
- cleanWhere[key] = value;
31
- }
32
- }
33
-
34
- for (const [key, value] of Object.entries(cleanWhere)) {
35
- if (key === 'OR' && Array.isArray(value)) {
36
- if (!value.some((v: any) => matchWhere(row, v))) return false;
37
- continue;
38
- }
39
- if (key === 'AND' && Array.isArray(value)) {
40
- if (!value.every((v: any) => matchWhere(row, v))) return false;
41
- continue;
42
- }
43
- if (key === 'NOT') {
44
- const notClauses = Array.isArray(value) ? value : [value];
45
- if (notClauses.some((v: any) => matchWhere(row, v))) return false;
46
- continue;
47
- }
48
-
49
- const cellVal = row[key];
50
-
51
- if (value === null) {
52
- if (cellVal !== null && cellVal !== undefined) return false;
53
- } else if (typeof value === 'object' && !(value instanceof Date)) {
54
- const v = value as any;
55
- const caseInsensitive = v.mode === 'insensitive';
56
- const normalize = (s: any) => caseInsensitive ? String(s ?? '').toLowerCase() : s;
57
-
58
- if (v.not !== undefined) {
59
- if (v.not === null) { if (cellVal === null || cellVal === undefined) return false; }
60
- else if (normalize(cellVal) == normalize(v.not)) return false;
61
- }
62
- if (v.equals !== undefined && normalize(cellVal) != normalize(v.equals)) return false;
63
- if (v.contains !== undefined) {
64
- if (!cellVal || !normalize(String(cellVal)).includes(normalize(v.contains))) return false;
65
- }
66
- if (v.startsWith !== undefined) {
67
- if (!cellVal || !normalize(String(cellVal)).startsWith(normalize(v.startsWith))) return false;
68
- }
69
- if (v.endsWith !== undefined) {
70
- if (!cellVal || !normalize(String(cellVal)).endsWith(normalize(v.endsWith))) return false;
71
- }
72
- if (v.gte !== undefined && !(Number(cellVal) >= Number(v.gte))) return false;
73
- if (v.lte !== undefined && !(Number(cellVal) <= Number(v.lte))) return false;
74
- if (v.gt !== undefined && !(Number(cellVal) > Number(v.gt))) return false;
75
- if (v.lt !== undefined && !(Number(cellVal) < Number(v.lt))) return false;
76
- if (v.in !== undefined) {
77
- if (!Array.isArray(v.in) || !v.in.some((item: any) => item == cellVal)) return false;
78
- }
79
- if (v.notIn !== undefined) {
80
- if (Array.isArray(v.notIn) && v.notIn.some((item: any) => item == cellVal)) return false;
81
- }
82
- } else {
83
- // loose equality to handle string/number coercion from sheet cells
84
- if (cellVal != value) return false;
85
- }
86
- }
87
- return true;
88
- }
89
-
90
-
91
- export function buildOrderBy(orderBy: any): string {
92
- if (!orderBy) return '';
93
- const entries = Array.isArray(orderBy) ? orderBy : [orderBy];
94
- const parts: string[] = [];
95
- for (const entry of entries) {
96
- for (const [key, dir] of Object.entries(entry)) {
97
- parts.push(`${key} ${String(dir || 'asc').toUpperCase()}`);
98
- }
99
- }
100
- return parts.join(', ');
101
- }
102
-
103
- export function sortRows(rows: Record<string, any>[], orderBy: string): Record<string, any>[] {
104
- if (!orderBy) return rows;
105
- const clauses = orderBy.split(',').map(c => c.trim());
106
- return [...rows].sort((a, b) => {
107
- for (const clause of clauses) {
108
- const [field, dir] = clause.split(/\s+/);
109
- const direction = dir?.toUpperCase() === 'DESC' ? -1 : 1;
110
- const aVal = a[field];
111
- const bVal = b[field];
112
- if (aVal == null && bVal == null) continue;
113
- if (aVal == null) return 1 * direction;
114
- if (bVal == null) return -1 * direction;
115
- if (aVal < bVal) return -1 * direction;
116
- if (aVal > bVal) return 1 * direction;
117
- }
118
- return 0;
119
- });
120
- }
121
-
122
- export function coerceCell(raw: any, fieldMeta?: { ts?: string; sql?: string }): any {
123
- if (raw === undefined || raw === '') return null;
124
- if (raw instanceof Date) return raw.toISOString();
125
- if (fieldMeta) {
126
- const tsType = (fieldMeta.ts || '').toLowerCase().replace('?', '');
127
- if (['number', 'int', 'float', 'double', 'decimal'].includes(tsType)) {
128
- const n = Number(raw);
129
- return isNaN(n) ? raw : n;
130
- }
131
- if (tsType === 'boolean' || tsType === 'bool') {
132
- if (raw === 'true' || raw === true) return true;
133
- if (raw === 'false' || raw === false) return false;
134
- }
135
- }
136
- if (typeof raw === 'string' && /^-?\d+\.?\d*$/.test(raw.trim()) && !/^0\d/.test(raw.trim())) {
137
- const n = Number(raw);
138
- if (!isNaN(n) && String(n) === raw.trim()) return n;
139
- }
140
- if (raw === 'true') return true;
141
- if (raw === 'false') return false;
142
- return raw;
143
- }
1
+ import { getModelToTable } from '../base/metadata';
2
+
3
+ // ─── Helpers ──────────────────────────────────────────────────────────────────
4
+
5
+ export function esc(name: string): string {
6
+ return name.includes(' ') || /[^\w]/.test(name) ? `'${name.replace(/'/g, "''")}'` : name;
7
+ }
8
+
9
+ export function resolveSheetName(modelName: string, mapping?: Record<string, string>): string {
10
+ if (mapping?.[modelName]) return mapping[modelName];
11
+ const modelToTable = getModelToTable();
12
+ return modelToTable[modelName] || modelName;
13
+ }
14
+
15
+ export function matchWhere(row: Record<string, any>, where: any): boolean {
16
+ if (!where) return true;
17
+
18
+ const cleanWhere: Record<string, any> = {};
19
+ for (const [key, value] of Object.entries(where)) {
20
+ if (
21
+ key.includes('_') &&
22
+ value && typeof value === 'object' &&
23
+ !(value instanceof Date) &&
24
+ !(value as any).in && !(value as any).contains &&
25
+ !(value as any).not && !(value as any).gte &&
26
+ !(value as any).lte && !(value as any).gt && !(value as any).lt
27
+ ) {
28
+ Object.assign(cleanWhere, value);
29
+ } else {
30
+ cleanWhere[key] = value;
31
+ }
32
+ }
33
+
34
+ for (const [key, value] of Object.entries(cleanWhere)) {
35
+ if (key === 'OR' && Array.isArray(value)) {
36
+ if (!value.some((v: any) => matchWhere(row, v))) return false;
37
+ continue;
38
+ }
39
+ if (key === 'AND' && Array.isArray(value)) {
40
+ if (!value.every((v: any) => matchWhere(row, v))) return false;
41
+ continue;
42
+ }
43
+ if (key === 'NOT') {
44
+ const notClauses = Array.isArray(value) ? value : [value];
45
+ if (notClauses.some((v: any) => matchWhere(row, v))) return false;
46
+ continue;
47
+ }
48
+
49
+ const cellVal = row[key];
50
+
51
+ if (value === null) {
52
+ if (cellVal !== null && cellVal !== undefined) return false;
53
+ } else if (typeof value === 'object' && !(value instanceof Date)) {
54
+ const v = value as any;
55
+ const caseInsensitive = v.mode === 'insensitive';
56
+ const normalize = (s: any) => caseInsensitive ? String(s ?? '').toLowerCase() : s;
57
+
58
+ if (v.not !== undefined) {
59
+ if (v.not === null) { if (cellVal === null || cellVal === undefined) return false; }
60
+ else if (normalize(cellVal) == normalize(v.not)) return false;
61
+ }
62
+ if (v.equals !== undefined && normalize(cellVal) != normalize(v.equals)) return false;
63
+ if (v.contains !== undefined) {
64
+ if (!cellVal || !normalize(String(cellVal)).includes(normalize(v.contains))) return false;
65
+ }
66
+ if (v.startsWith !== undefined) {
67
+ if (!cellVal || !normalize(String(cellVal)).startsWith(normalize(v.startsWith))) return false;
68
+ }
69
+ if (v.endsWith !== undefined) {
70
+ if (!cellVal || !normalize(String(cellVal)).endsWith(normalize(v.endsWith))) return false;
71
+ }
72
+ if (v.gte !== undefined && !(Number(cellVal) >= Number(v.gte))) return false;
73
+ if (v.lte !== undefined && !(Number(cellVal) <= Number(v.lte))) return false;
74
+ if (v.gt !== undefined && !(Number(cellVal) > Number(v.gt))) return false;
75
+ if (v.lt !== undefined && !(Number(cellVal) < Number(v.lt))) return false;
76
+ if (v.in !== undefined) {
77
+ if (!Array.isArray(v.in) || !v.in.some((item: any) => item == cellVal)) return false;
78
+ }
79
+ if (v.notIn !== undefined) {
80
+ if (Array.isArray(v.notIn) && v.notIn.some((item: any) => item == cellVal)) return false;
81
+ }
82
+ } else {
83
+ // loose equality to handle string/number coercion from sheet cells
84
+ if (cellVal != value) return false;
85
+ }
86
+ }
87
+ return true;
88
+ }
89
+
90
+
91
+ export function buildOrderBy(orderBy: any): string {
92
+ if (!orderBy) return '';
93
+ const entries = Array.isArray(orderBy) ? orderBy : [orderBy];
94
+ const parts: string[] = [];
95
+ for (const entry of entries) {
96
+ for (const [key, dir] of Object.entries(entry)) {
97
+ parts.push(`${key} ${String(dir || 'asc').toUpperCase()}`);
98
+ }
99
+ }
100
+ return parts.join(', ');
101
+ }
102
+
103
+ export function sortRows(rows: Record<string, any>[], orderBy: string): Record<string, any>[] {
104
+ if (!orderBy) return rows;
105
+ const clauses = orderBy.split(',').map(c => c.trim());
106
+ return [...rows].sort((a, b) => {
107
+ for (const clause of clauses) {
108
+ const [field, dir] = clause.split(/\s+/);
109
+ if (field === undefined) continue;
110
+ const direction = dir?.toUpperCase() === 'DESC' ? -1 : 1;
111
+ const aVal = a[field];
112
+ const bVal = b[field];
113
+ if (aVal == null && bVal == null) continue;
114
+ if (aVal == null) return 1 * direction;
115
+ if (bVal == null) return -1 * direction;
116
+ if (aVal < bVal) return -1 * direction;
117
+ if (aVal > bVal) return 1 * direction;
118
+ }
119
+ return 0;
120
+ });
121
+ }
122
+
123
+ export function coerceCell(raw: any, fieldMeta?: { ts?: string; sql?: string }): any {
124
+ if (raw === undefined || raw === '') return null;
125
+ if (raw instanceof Date) return raw.toISOString();
126
+ if (fieldMeta) {
127
+ const tsType = (fieldMeta.ts || '').toLowerCase().replace('?', '');
128
+ if (['number', 'int', 'float', 'double', 'decimal'].includes(tsType)) {
129
+ const n = Number(raw);
130
+ return isNaN(n) ? raw : n;
131
+ }
132
+ if (tsType === 'boolean' || tsType === 'bool') {
133
+ if (raw === 'true' || raw === true) return true;
134
+ if (raw === 'false' || raw === false) return false;
135
+ }
136
+ }
137
+ if (typeof raw === 'string' && /^-?\d+\.?\d*$/.test(raw.trim()) && !/^0\d/.test(raw.trim())) {
138
+ const n = Number(raw);
139
+ if (!isNaN(n) && String(n) === raw.trim()) return n;
140
+ }
141
+ if (raw === 'true') return true;
142
+ if (raw === 'false') return false;
143
+ return raw;
144
+ }
@@ -1,3 +1,3 @@
1
- export { An5SheetsAdapter, SheetsTableClient, createAn5SheetsAdapter } from './an5SheetsAdapter';
2
- export type { An5SheetsAdapterConfig } from './an5SheetsAdapter';
3
- export { parseSheetsConnectionString } from './parseConnectionString';
1
+ export { An5SheetsAdapter, SheetsTableClient, createAn5SheetsAdapter } from './an5SheetsAdapter';
2
+ export type { An5SheetsAdapterConfig } from './an5SheetsAdapter';
3
+ export { parseSheetsConnectionString } from './parseConnectionString';
@@ -1,46 +1,48 @@
1
- import type { An5SheetsAdapterConfig } from './config';
2
-
3
- /**
4
- * Parse a `googlesheets://` connection string into An5SheetsAdapterConfig.
5
- *
6
- * Format:
7
- * googlesheets://spreadsheetId;clientEmail=xxx;privateKey=xxx
8
- * googlesheets://spreadsheetId;clientEmail=xxx;privateKey=xxx;sheetMapping=users:Users,orders:Orders
9
- *
10
- * The privateKey must be URL-encoded (use encodeURIComponent on the raw key).
11
- */
12
- export function parseSheetsConnectionString(url: string): An5SheetsAdapterConfig {
13
- const clean = url.replace(/^googlesheets:\/\//, '');
14
- const parts = clean.split(';');
15
-
16
- const spreadsheetId = parts[0].trim();
17
- const config: Record<string, string> = {};
18
-
19
- for (let i = 1; i < parts.length; i++) {
20
- const part = parts[i].trim();
21
- if (!part) continue;
22
- const eqIdx = part.indexOf('=');
23
- if (eqIdx === -1) continue;
24
- const key = part.slice(0, eqIdx).trim().toLowerCase();
25
- const value = decodeURIComponent(part.slice(eqIdx + 1).trim());
26
- config[key] = value;
27
- }
28
-
29
- const result: An5SheetsAdapterConfig = { spreadsheetId };
30
-
31
- if (config.clientemail) result.clientEmail = config.clientemail;
32
- if (config.privatekey) result.privateKey = config.privatekey;
33
- if (config.accesstoken) result.accessToken = config.accesstoken;
34
- if (config.apikey) result.apiKey = config.apikey;
35
-
36
- if (config.sheetmapping) {
37
- const mapping: Record<string, string> = {};
38
- for (const pair of config.sheetmapping.split(',')) {
39
- const [k, v] = pair.split(':');
40
- if (k && v) mapping[k.trim()] = v.trim();
41
- }
42
- result.sheetMapping = mapping;
43
- }
44
-
45
- return result;
46
- }
1
+ import type { An5SheetsAdapterConfig } from './config';
2
+
3
+ /**
4
+ * Parse a `googlesheets://` connection string into An5SheetsAdapterConfig.
5
+ *
6
+ * Format:
7
+ * googlesheets://spreadsheetId;clientEmail=xxx;privateKey=xxx
8
+ * googlesheets://spreadsheetId;clientEmail=xxx;privateKey=xxx;sheetMapping=users:Users,orders:Orders
9
+ *
10
+ * The privateKey must be URL-encoded (use encodeURIComponent on the raw key).
11
+ */
12
+ export function parseSheetsConnectionString(url: string): An5SheetsAdapterConfig {
13
+ const clean = url.replace(/^googlesheets:\/\//, '');
14
+ const parts = clean.split(';');
15
+
16
+ const spreadsheetId = (parts[0] ?? '').trim();
17
+ const config: Record<string, string> = {};
18
+
19
+ for (let i = 1; i < parts.length; i++) {
20
+ const rawPart = parts[i];
21
+ if (rawPart === undefined) continue;
22
+ const part = rawPart.trim();
23
+ if (!part) continue;
24
+ const eqIdx = part.indexOf('=');
25
+ if (eqIdx === -1) continue;
26
+ const key = part.slice(0, eqIdx).trim().toLowerCase();
27
+ const value = decodeURIComponent(part.slice(eqIdx + 1).trim());
28
+ config[key] = value;
29
+ }
30
+
31
+ const result: An5SheetsAdapterConfig = { spreadsheetId };
32
+
33
+ if (config.clientemail) result.clientEmail = config.clientemail;
34
+ if (config.privatekey) result.privateKey = config.privatekey;
35
+ if (config.accesstoken) result.accessToken = config.accesstoken;
36
+ if (config.apikey) result.apiKey = config.apikey;
37
+
38
+ if (config.sheetmapping) {
39
+ const mapping: Record<string, string> = {};
40
+ for (const pair of config.sheetmapping.split(',')) {
41
+ const [k, v] = pair.split(':');
42
+ if (k && v) mapping[k.trim()] = v.trim();
43
+ }
44
+ result.sheetMapping = mapping;
45
+ }
46
+
47
+ return result;
48
+ }
@@ -1,126 +1,126 @@
1
- // ─── Rate Limit & Retry helper ─────────────────────────────────────────────────
2
- //
3
- // Google Sheets API quota (default): 100 requests per 100 seconds per project.
4
- // Strategy:
5
- // 1. Token-bucket rate limiter — proactively throttles before hitting quota
6
- // 2. Concurrency limiter — max N parallel requests to avoid burst
7
- // 3. Exponential backoff + jitter — reactive recovery when 429/403/5xx hit
8
-
9
- // ─── Config ────────────────────────────────────────────────────────────────────
10
-
11
- export const MAX_RETRIES = 5;
12
- export const BASE_DELAY = 1000; // ms
13
- export const MAX_DELAY = 32_000; // ms cap for backoff
14
- export const MAX_CONCURRENCY = 4; // parallel in-flight requests
15
- export const RATE_LIMIT_RPS = 0.9; // tokens/second (stay under 1 req/s safely)
16
-
17
- // ─── Token Bucket ──────────────────────────────────────────────────────────────
18
-
19
- class TokenBucket {
20
- private tokens: number;
21
- private lastRefill: number;
22
- private readonly ratePerMs: number;
23
- private readonly capacity: number;
24
-
25
- constructor(ratePerSecond: number, capacity: number) {
26
- this.ratePerMs = ratePerSecond / 1000;
27
- this.capacity = capacity;
28
- this.tokens = capacity;
29
- this.lastRefill = Date.now();
30
- }
31
-
32
- /** Wait until a token is available, then consume one. */
33
- async acquire(): Promise<void> {
34
- while (true) {
35
- const now = Date.now();
36
- const elapsed = now - this.lastRefill;
37
- this.tokens = Math.min(this.capacity, this.tokens + elapsed * this.ratePerMs);
38
- this.lastRefill = now;
39
-
40
- if (this.tokens >= 1) {
41
- this.tokens -= 1;
42
- return;
43
- }
44
-
45
- // Wait for enough time to refill one token
46
- const waitMs = Math.ceil((1 - this.tokens) / this.ratePerMs);
47
- await new Promise(r => setTimeout(r, waitMs));
48
- }
49
- }
50
- }
51
-
52
- // ─── Concurrency Limiter ───────────────────────────────────────────────────────
53
-
54
- class ConcurrencyLimiter {
55
- private running = 0;
56
- private queue: Array<() => void> = [];
57
-
58
- constructor(private readonly max: number) {}
59
-
60
- async acquire(): Promise<void> {
61
- if (this.running < this.max) {
62
- this.running++;
63
- return;
64
- }
65
- await new Promise<void>(resolve => this.queue.push(resolve));
66
- this.running++;
67
- }
68
-
69
- release(): void {
70
- this.running--;
71
- if (this.queue.length > 0) {
72
- const next = this.queue.shift()!;
73
- next();
74
- }
75
- }
76
- }
77
-
78
- // ─── Singleton shared across all adapter instances ─────────────────────────────
79
-
80
- const _bucket = new TokenBucket(RATE_LIMIT_RPS, 10);
81
- const _concurrency = new ConcurrencyLimiter(MAX_CONCURRENCY);
82
-
83
- // ─── Error classification ──────────────────────────────────────────────────────
84
-
85
- function isRetryableError(err: any): boolean {
86
- const code = err?.code || err?.status || err?.response?.status;
87
- if (code === 429) return true;
88
- if (code === 500 || code === 503 || code === 502) return true;
89
- if (code === 403) {
90
- const reason = err?.errors?.[0]?.reason || '';
91
- if (reason === 'rateLimitExceeded' || reason === 'userRateLimitExceeded') return true;
92
- if (typeof err?.body === 'string' && err.body.includes('rateLimitExceeded')) return true;
93
- if (typeof err?.message === 'string' && err.message.includes('rateLimitExceeded')) return true;
94
- }
95
- return false;
96
- }
97
-
98
- function getRetryDelay(err: any, attempt: number): number {
99
- // Honour server-provided Retry-After
100
- if (err?.retryAfter) return err.retryAfter * 1000;
101
- // Full jitter exponential backoff: random in [0, min(cap, base * 2^attempt)]
102
- const cap = Math.min(MAX_DELAY, BASE_DELAY * Math.pow(2, attempt));
103
- return Math.random() * cap;
104
- }
105
-
106
- // ─── Public API ────────────────────────────────────────────────────────────────
107
-
108
- export async function withRetry<T>(fn: () => Promise<T>, attempt = 1): Promise<T> {
109
- await _bucket.acquire();
110
- await _concurrency.acquire();
111
- try {
112
- return await fn();
113
- } catch (err: any) {
114
- if (isRetryableError(err) && attempt <= MAX_RETRIES) {
115
- const delay = getRetryDelay(err, attempt);
116
- await new Promise(r => setTimeout(r, delay));
117
- return withRetry(fn, attempt + 1);
118
- }
119
- throw err;
120
- } finally {
121
- _concurrency.release();
122
- }
123
- }
124
-
125
- /** Expose for testing / custom configurations */
126
- export { TokenBucket, ConcurrencyLimiter };
1
+ // ─── Rate Limit & Retry helper ─────────────────────────────────────────────────
2
+ //
3
+ // Google Sheets API quota (default): 100 requests per 100 seconds per project.
4
+ // Strategy:
5
+ // 1. Token-bucket rate limiter — proactively throttles before hitting quota
6
+ // 2. Concurrency limiter — max N parallel requests to avoid burst
7
+ // 3. Exponential backoff + jitter — reactive recovery when 429/403/5xx hit
8
+
9
+ // ─── Config ────────────────────────────────────────────────────────────────────
10
+
11
+ export const MAX_RETRIES = 5;
12
+ export const BASE_DELAY = 1000; // ms
13
+ export const MAX_DELAY = 32_000; // ms cap for backoff
14
+ export const MAX_CONCURRENCY = 4; // parallel in-flight requests
15
+ export const RATE_LIMIT_RPS = 0.9; // tokens/second (stay under 1 req/s safely)
16
+
17
+ // ─── Token Bucket ──────────────────────────────────────────────────────────────
18
+
19
+ class TokenBucket {
20
+ private tokens: number;
21
+ private lastRefill: number;
22
+ private readonly ratePerMs: number;
23
+ private readonly capacity: number;
24
+
25
+ constructor(ratePerSecond: number, capacity: number) {
26
+ this.ratePerMs = ratePerSecond / 1000;
27
+ this.capacity = capacity;
28
+ this.tokens = capacity;
29
+ this.lastRefill = Date.now();
30
+ }
31
+
32
+ /** Wait until a token is available, then consume one. */
33
+ async acquire(): Promise<void> {
34
+ while (true) {
35
+ const now = Date.now();
36
+ const elapsed = now - this.lastRefill;
37
+ this.tokens = Math.min(this.capacity, this.tokens + elapsed * this.ratePerMs);
38
+ this.lastRefill = now;
39
+
40
+ if (this.tokens >= 1) {
41
+ this.tokens -= 1;
42
+ return;
43
+ }
44
+
45
+ // Wait for enough time to refill one token
46
+ const waitMs = Math.ceil((1 - this.tokens) / this.ratePerMs);
47
+ await new Promise(r => setTimeout(r, waitMs));
48
+ }
49
+ }
50
+ }
51
+
52
+ // ─── Concurrency Limiter ───────────────────────────────────────────────────────
53
+
54
+ class ConcurrencyLimiter {
55
+ private running = 0;
56
+ private queue: Array<() => void> = [];
57
+
58
+ constructor(private readonly max: number) {}
59
+
60
+ async acquire(): Promise<void> {
61
+ if (this.running < this.max) {
62
+ this.running++;
63
+ return;
64
+ }
65
+ await new Promise<void>(resolve => this.queue.push(resolve));
66
+ this.running++;
67
+ }
68
+
69
+ release(): void {
70
+ this.running--;
71
+ if (this.queue.length > 0) {
72
+ const next = this.queue.shift()!;
73
+ next();
74
+ }
75
+ }
76
+ }
77
+
78
+ // ─── Singleton shared across all adapter instances ─────────────────────────────
79
+
80
+ const _bucket = new TokenBucket(RATE_LIMIT_RPS, 10);
81
+ const _concurrency = new ConcurrencyLimiter(MAX_CONCURRENCY);
82
+
83
+ // ─── Error classification ──────────────────────────────────────────────────────
84
+
85
+ function isRetryableError(err: any): boolean {
86
+ const code = err?.code || err?.status || err?.response?.status;
87
+ if (code === 429) return true;
88
+ if (code === 500 || code === 503 || code === 502) return true;
89
+ if (code === 403) {
90
+ const reason = err?.errors?.[0]?.reason || '';
91
+ if (reason === 'rateLimitExceeded' || reason === 'userRateLimitExceeded') return true;
92
+ if (typeof err?.body === 'string' && err.body.includes('rateLimitExceeded')) return true;
93
+ if (typeof err?.message === 'string' && err.message.includes('rateLimitExceeded')) return true;
94
+ }
95
+ return false;
96
+ }
97
+
98
+ function getRetryDelay(err: any, attempt: number): number {
99
+ // Honour server-provided Retry-After
100
+ if (err?.retryAfter) return err.retryAfter * 1000;
101
+ // Full jitter exponential backoff: random in [0, min(cap, base * 2^attempt)]
102
+ const cap = Math.min(MAX_DELAY, BASE_DELAY * Math.pow(2, attempt));
103
+ return Math.random() * cap;
104
+ }
105
+
106
+ // ─── Public API ────────────────────────────────────────────────────────────────
107
+
108
+ export async function withRetry<T>(fn: () => Promise<T>, attempt = 1): Promise<T> {
109
+ await _bucket.acquire();
110
+ await _concurrency.acquire();
111
+ try {
112
+ return await fn();
113
+ } catch (err: any) {
114
+ if (isRetryableError(err) && attempt <= MAX_RETRIES) {
115
+ const delay = getRetryDelay(err, attempt);
116
+ await new Promise(r => setTimeout(r, delay));
117
+ return withRetry(fn, attempt + 1);
118
+ }
119
+ throw err;
120
+ } finally {
121
+ _concurrency.release();
122
+ }
123
+ }
124
+
125
+ /** Expose for testing / custom configurations */
126
+ export { TokenBucket, ConcurrencyLimiter };