@withone/cli 1.29.0 → 1.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +69 -2
- package/dist/{chunk-AZV4EGKT.js → chunk-645Z3ARQ.js} +124 -32
- package/dist/{flow-runner-AK5W4GLF.js → flow-runner-HT74FKPD.js} +1 -1
- package/dist/index.js +4074 -218
- package/package.json +8 -3
- package/profiles/attio/attioCompanies.json +16 -0
- package/profiles/attio/attioPeople.json +17 -0
- package/profiles/fathom/meetings.json +27 -0
- package/profiles/gmail/gmailThreads.json +28 -0
- package/profiles/google-calendar/events.json +17 -0
- package/profiles/notion/search.json +18 -0
- package/profiles/stripe/balanceTransactions.json +17 -0
- package/profiles/stripe/customers.json +18 -0
- package/skills/one/SKILL.md +43 -0
- package/skills/one/references/flows.md +4 -0
package/dist/index.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
|
+
ApiError,
|
|
3
4
|
FLOW_SCHEMA,
|
|
4
5
|
FlowRunner,
|
|
5
6
|
OneApi,
|
|
@@ -8,6 +9,7 @@ import {
|
|
|
8
9
|
filterByPermissions,
|
|
9
10
|
flowRequiresBash,
|
|
10
11
|
generateFlowGuide,
|
|
12
|
+
getByDotPath,
|
|
11
13
|
getNestedStepsKeys,
|
|
12
14
|
getStepTypeDescriptor,
|
|
13
15
|
isActionAllowed,
|
|
@@ -15,8 +17,9 @@ import {
|
|
|
15
17
|
listFlows,
|
|
16
18
|
loadFlowWithMeta,
|
|
17
19
|
resolveFlowPath,
|
|
18
|
-
saveFlow
|
|
19
|
-
|
|
20
|
+
saveFlow,
|
|
21
|
+
validateActionInput
|
|
22
|
+
} from "./chunk-645Z3ARQ.js";
|
|
20
23
|
|
|
21
24
|
// src/index.ts
|
|
22
25
|
import { createRequire as createRequire2 } from "module";
|
|
@@ -224,11 +227,11 @@ import fs2 from "fs";
|
|
|
224
227
|
import path2 from "path";
|
|
225
228
|
import os2 from "os";
|
|
226
229
|
import { parse as parseToml, stringify as stringifyToml } from "smol-toml";
|
|
227
|
-
function expandPath(
|
|
228
|
-
if (
|
|
229
|
-
return path2.join(os2.homedir(),
|
|
230
|
+
function expandPath(p8) {
|
|
231
|
+
if (p8.startsWith("~/")) {
|
|
232
|
+
return path2.join(os2.homedir(), p8.slice(2));
|
|
230
233
|
}
|
|
231
|
-
return
|
|
234
|
+
return p8;
|
|
232
235
|
}
|
|
233
236
|
function getClaudeDesktopConfigPath() {
|
|
234
237
|
switch (process.platform) {
|
|
@@ -444,6 +447,9 @@ function outro2(msg) {
|
|
|
444
447
|
function note2(msg, title) {
|
|
445
448
|
if (!isAgentMode()) p.note(msg, title);
|
|
446
449
|
}
|
|
450
|
+
function cancel2(msg) {
|
|
451
|
+
if (!isAgentMode()) p.cancel(msg);
|
|
452
|
+
}
|
|
447
453
|
function json(data) {
|
|
448
454
|
process.stdout.write(JSON.stringify(data) + "\n");
|
|
449
455
|
}
|
|
@@ -1506,16 +1512,16 @@ import pc4 from "picocolors";
|
|
|
1506
1512
|
function findPlatform(platforms, query) {
|
|
1507
1513
|
const normalizedQuery = query.toLowerCase().trim();
|
|
1508
1514
|
const exact = platforms.find(
|
|
1509
|
-
(
|
|
1515
|
+
(p8) => p8.platform.toLowerCase() === normalizedQuery || p8.name.toLowerCase() === normalizedQuery
|
|
1510
1516
|
);
|
|
1511
1517
|
if (exact) return exact;
|
|
1512
1518
|
return null;
|
|
1513
1519
|
}
|
|
1514
1520
|
function findSimilarPlatforms(platforms, query, limit = 3) {
|
|
1515
1521
|
const normalizedQuery = query.toLowerCase().trim();
|
|
1516
|
-
const scored = platforms.map((
|
|
1517
|
-
const name =
|
|
1518
|
-
const slug =
|
|
1522
|
+
const scored = platforms.map((p8) => {
|
|
1523
|
+
const name = p8.name.toLowerCase();
|
|
1524
|
+
const slug = p8.platform.toLowerCase();
|
|
1519
1525
|
let score = 0;
|
|
1520
1526
|
if (name.includes(normalizedQuery) || slug.includes(normalizedQuery)) {
|
|
1521
1527
|
score = 10;
|
|
@@ -1524,7 +1530,7 @@ function findSimilarPlatforms(platforms, query, limit = 3) {
|
|
|
1524
1530
|
} else {
|
|
1525
1531
|
score = countMatchingChars(normalizedQuery, slug);
|
|
1526
1532
|
}
|
|
1527
|
-
return { platform:
|
|
1533
|
+
return { platform: p8, score };
|
|
1528
1534
|
}).filter((item) => item.score > 0).sort((a, b) => b.score - a.score).slice(0, limit);
|
|
1529
1535
|
return scored.map((item) => item.platform);
|
|
1530
1536
|
}
|
|
@@ -1849,35 +1855,33 @@ async function platformsCommand(options) {
|
|
|
1849
1855
|
try {
|
|
1850
1856
|
const platforms = await api.listPlatforms();
|
|
1851
1857
|
spinner5.stop(`${platforms.length} platforms available`);
|
|
1858
|
+
let filtered = platforms;
|
|
1859
|
+
if (options.category) {
|
|
1860
|
+
filtered = platforms.filter((plat) => (plat.category || "Other") === options.category);
|
|
1861
|
+
if (filtered.length === 0) {
|
|
1862
|
+
const categories = [...new Set(platforms.map((plat) => plat.category || "Other"))].sort();
|
|
1863
|
+
if (isAgentMode()) {
|
|
1864
|
+
json({ error: `Unknown category "${options.category}"`, availableCategories: categories });
|
|
1865
|
+
process.exit(1);
|
|
1866
|
+
}
|
|
1867
|
+
p5.note(`Available categories:
|
|
1868
|
+
${categories.join(", ")}`, "Unknown Category");
|
|
1869
|
+
process.exit(1);
|
|
1870
|
+
}
|
|
1871
|
+
}
|
|
1852
1872
|
if (options.json) {
|
|
1853
1873
|
if (isAgentMode()) {
|
|
1854
|
-
json({ platforms });
|
|
1874
|
+
json({ platforms: filtered });
|
|
1855
1875
|
} else {
|
|
1856
|
-
console.log(JSON.stringify(
|
|
1876
|
+
console.log(JSON.stringify(filtered, null, 2));
|
|
1857
1877
|
}
|
|
1858
1878
|
return;
|
|
1859
1879
|
}
|
|
1860
|
-
const byCategory = /* @__PURE__ */ new Map();
|
|
1861
|
-
for (const plat of platforms) {
|
|
1862
|
-
const category = plat.category || "Other";
|
|
1863
|
-
if (!byCategory.has(category)) {
|
|
1864
|
-
byCategory.set(category, []);
|
|
1865
|
-
}
|
|
1866
|
-
byCategory.get(category).push(plat);
|
|
1867
|
-
}
|
|
1868
1880
|
console.log();
|
|
1869
1881
|
if (options.category) {
|
|
1870
|
-
const
|
|
1871
|
-
if (!categoryPlatforms) {
|
|
1872
|
-
const categories = [...byCategory.keys()].sort();
|
|
1873
|
-
p5.note(`Available categories:
|
|
1874
|
-
${categories.join(", ")}`, "Unknown Category");
|
|
1875
|
-
process.exit(1);
|
|
1876
|
-
}
|
|
1877
|
-
const rows = categoryPlatforms.sort((a, b) => a.platform.localeCompare(b.platform)).map((plat) => ({
|
|
1882
|
+
const rows = filtered.sort((a, b) => a.platform.localeCompare(b.platform)).map((plat) => ({
|
|
1878
1883
|
platform: plat.platform,
|
|
1879
|
-
name: plat.name
|
|
1880
|
-
category: plat.category || "Other"
|
|
1884
|
+
name: plat.name
|
|
1881
1885
|
}));
|
|
1882
1886
|
printTable(
|
|
1883
1887
|
[
|
|
@@ -1887,7 +1891,7 @@ async function platformsCommand(options) {
|
|
|
1887
1891
|
rows
|
|
1888
1892
|
);
|
|
1889
1893
|
} else {
|
|
1890
|
-
const rows =
|
|
1894
|
+
const rows = filtered.sort((a, b) => (a.category || "Other").localeCompare(b.category || "Other") || a.platform.localeCompare(b.platform)).map((plat) => ({
|
|
1891
1895
|
platform: plat.platform,
|
|
1892
1896
|
name: plat.name,
|
|
1893
1897
|
category: plat.category || "Other"
|
|
@@ -2058,11 +2062,11 @@ async function actionsSearchCommand(platform, query, options) {
|
|
|
2058
2062
|
const agentType = knowledgeAgent ? "knowledge" : options.type || "execute";
|
|
2059
2063
|
const useCache = options.cache !== false;
|
|
2060
2064
|
const cachePath = searchCachePath(platform, query, agentType || "knowledge");
|
|
2061
|
-
const
|
|
2065
|
+
const cached2 = useCache ? readCache2(cachePath) : null;
|
|
2062
2066
|
let cleanedActions;
|
|
2063
2067
|
let cacheHit = false;
|
|
2064
|
-
if (
|
|
2065
|
-
cleanedActions =
|
|
2068
|
+
if (cached2 && isFresh(cached2)) {
|
|
2069
|
+
cleanedActions = cached2.data.actions;
|
|
2066
2070
|
cacheHit = true;
|
|
2067
2071
|
} else {
|
|
2068
2072
|
try {
|
|
@@ -2070,12 +2074,12 @@ async function actionsSearchCommand(platform, query, options) {
|
|
|
2070
2074
|
platform,
|
|
2071
2075
|
query,
|
|
2072
2076
|
agentType,
|
|
2073
|
-
|
|
2077
|
+
cached2?.etag ?? void 0
|
|
2074
2078
|
);
|
|
2075
|
-
if (result.status === 304 &&
|
|
2076
|
-
|
|
2077
|
-
writeCache2(cachePath,
|
|
2078
|
-
cleanedActions =
|
|
2079
|
+
if (result.status === 304 && cached2) {
|
|
2080
|
+
cached2.cachedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
2081
|
+
writeCache2(cachePath, cached2);
|
|
2082
|
+
cleanedActions = cached2.data.actions;
|
|
2079
2083
|
cacheHit = true;
|
|
2080
2084
|
} else {
|
|
2081
2085
|
let actions2 = result.data;
|
|
@@ -2094,12 +2098,12 @@ async function actionsSearchCommand(platform, query, options) {
|
|
|
2094
2098
|
));
|
|
2095
2099
|
}
|
|
2096
2100
|
} catch (fetchError) {
|
|
2097
|
-
if (
|
|
2101
|
+
if (cached2) {
|
|
2098
2102
|
process.stderr.write(
|
|
2099
|
-
`Warning: serving cached search results (network unavailable, cached ${formatAge(getAge(
|
|
2103
|
+
`Warning: serving cached search results (network unavailable, cached ${formatAge(getAge(cached2))} ago)
|
|
2100
2104
|
`
|
|
2101
2105
|
);
|
|
2102
|
-
cleanedActions =
|
|
2106
|
+
cleanedActions = cached2.data.actions;
|
|
2103
2107
|
cacheHit = true;
|
|
2104
2108
|
} else {
|
|
2105
2109
|
throw fetchError;
|
|
@@ -2108,8 +2112,8 @@ async function actionsSearchCommand(platform, query, options) {
|
|
|
2108
2112
|
}
|
|
2109
2113
|
if (isAgentMode()) {
|
|
2110
2114
|
const response = { actions: cleanedActions };
|
|
2111
|
-
if (cacheHit &&
|
|
2112
|
-
response._cache = buildCacheMeta(
|
|
2115
|
+
if (cacheHit && cached2) {
|
|
2116
|
+
response._cache = buildCacheMeta(cached2, true);
|
|
2113
2117
|
} else {
|
|
2114
2118
|
const freshEntry = readCache2(cachePath);
|
|
2115
2119
|
response._cache = buildCacheMeta(freshEntry, false);
|
|
@@ -2219,23 +2223,23 @@ async function actionsKnowledgeCommand(platform, actionId, options) {
|
|
|
2219
2223
|
spinner5.start(`Loading knowledge for action ${pc6.dim(actionId)}...`);
|
|
2220
2224
|
try {
|
|
2221
2225
|
const useCache = options.cache !== false;
|
|
2222
|
-
const
|
|
2226
|
+
const cached2 = useCache ? readCache2(cachePath) : null;
|
|
2223
2227
|
let knowledgeData;
|
|
2224
2228
|
let cacheHit = false;
|
|
2225
|
-
let cacheEntry =
|
|
2226
|
-
if (
|
|
2227
|
-
knowledgeData =
|
|
2229
|
+
let cacheEntry = cached2;
|
|
2230
|
+
if (cached2 && isFresh(cached2) && useCache) {
|
|
2231
|
+
knowledgeData = cached2.data;
|
|
2228
2232
|
cacheHit = true;
|
|
2229
2233
|
} else {
|
|
2230
2234
|
try {
|
|
2231
2235
|
const result = await api.getActionKnowledgeWithMeta(
|
|
2232
2236
|
actionId,
|
|
2233
|
-
|
|
2237
|
+
cached2?.etag ?? void 0
|
|
2234
2238
|
);
|
|
2235
|
-
if (result.status === 304 &&
|
|
2236
|
-
|
|
2237
|
-
writeCache2(cachePath,
|
|
2238
|
-
knowledgeData =
|
|
2239
|
+
if (result.status === 304 && cached2) {
|
|
2240
|
+
cached2.cachedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
2241
|
+
writeCache2(cachePath, cached2);
|
|
2242
|
+
knowledgeData = cached2.data;
|
|
2239
2243
|
cacheHit = true;
|
|
2240
2244
|
} else {
|
|
2241
2245
|
knowledgeData = result.data;
|
|
@@ -2244,12 +2248,12 @@ async function actionsKnowledgeCommand(platform, actionId, options) {
|
|
|
2244
2248
|
cacheEntry = newEntry;
|
|
2245
2249
|
}
|
|
2246
2250
|
} catch (fetchError) {
|
|
2247
|
-
if (
|
|
2251
|
+
if (cached2) {
|
|
2248
2252
|
process.stderr.write(
|
|
2249
|
-
`Warning: serving cached knowledge (network unavailable, cached ${formatAge(getAge(
|
|
2253
|
+
`Warning: serving cached knowledge (network unavailable, cached ${formatAge(getAge(cached2))} ago)
|
|
2250
2254
|
`
|
|
2251
2255
|
);
|
|
2252
|
-
knowledgeData =
|
|
2256
|
+
knowledgeData = cached2.data;
|
|
2253
2257
|
cacheHit = true;
|
|
2254
2258
|
} else {
|
|
2255
2259
|
throw fetchError;
|
|
@@ -2316,6 +2320,53 @@ async function actionsExecuteCommand(platform, actionId, connectionKey, options)
|
|
|
2316
2320
|
const pathVariables = options.pathVars ? parseJsonArg(options.pathVars, "--path-vars") : void 0;
|
|
2317
2321
|
const queryParams = options.queryParams ? parseJsonArg(options.queryParams, "--query-params") : void 0;
|
|
2318
2322
|
const headers = options.headers ? parseJsonArg(options.headers, "--headers") : void 0;
|
|
2323
|
+
if (!options.skipValidation) {
|
|
2324
|
+
const validation = validateActionInput(actionDetails, { data, pathVariables, queryParams });
|
|
2325
|
+
if (!validation.valid) {
|
|
2326
|
+
spinner5.stop("Validation failed");
|
|
2327
|
+
if (isAgentMode()) {
|
|
2328
|
+
json({
|
|
2329
|
+
error: "Validation failed: missing required parameters",
|
|
2330
|
+
validation: { missing: validation.missing },
|
|
2331
|
+
hint: "Add the missing parameters, or pass --skip-validation to bypass this check."
|
|
2332
|
+
});
|
|
2333
|
+
process.exit(1);
|
|
2334
|
+
}
|
|
2335
|
+
console.log();
|
|
2336
|
+
for (const m of validation.missing) {
|
|
2337
|
+
console.log(pc6.red(` ${m.flag} is missing "${m.param}"`));
|
|
2338
|
+
if (m.description) {
|
|
2339
|
+
console.log(pc6.dim(` ${m.description}`));
|
|
2340
|
+
}
|
|
2341
|
+
}
|
|
2342
|
+
console.log();
|
|
2343
|
+
error("Validation failed: missing required parameters. Pass --skip-validation to bypass.");
|
|
2344
|
+
}
|
|
2345
|
+
}
|
|
2346
|
+
if (options.mock) {
|
|
2347
|
+
spinner5.stop("Mock \u2014 returning example response");
|
|
2348
|
+
const mockResponse = actionDetails.ioSchema?.ioExample?.output ?? null;
|
|
2349
|
+
if (isAgentMode()) {
|
|
2350
|
+
json({
|
|
2351
|
+
mock: true,
|
|
2352
|
+
request: {
|
|
2353
|
+
method: actionDetails.method,
|
|
2354
|
+
url: actionDetails.path
|
|
2355
|
+
},
|
|
2356
|
+
response: mockResponse,
|
|
2357
|
+
...mockResponse === null ? { message: "No example output available for this action" } : {}
|
|
2358
|
+
});
|
|
2359
|
+
return;
|
|
2360
|
+
}
|
|
2361
|
+
console.log();
|
|
2362
|
+
if (mockResponse) {
|
|
2363
|
+
console.log(pc6.bold("Mock Response:"));
|
|
2364
|
+
console.log(JSON.stringify(mockResponse, null, 2));
|
|
2365
|
+
} else {
|
|
2366
|
+
note2("No example output available for this action", "Mock");
|
|
2367
|
+
}
|
|
2368
|
+
return;
|
|
2369
|
+
}
|
|
2319
2370
|
const execSpinner = createSpinner();
|
|
2320
2371
|
execSpinner.start(options.dryRun ? "Building request..." : "Executing action...");
|
|
2321
2372
|
const result = await api.executePassthroughRequest(
|
|
@@ -2374,6 +2425,293 @@ async function actionsExecuteCommand(platform, actionId, connectionKey, options)
|
|
|
2374
2425
|
);
|
|
2375
2426
|
}
|
|
2376
2427
|
}
|
|
2428
|
+
function parseParallelSegments() {
|
|
2429
|
+
const argv = process.argv.slice(2);
|
|
2430
|
+
let execIdx = -1;
|
|
2431
|
+
for (let i = 0; i < argv.length; i++) {
|
|
2432
|
+
if ((argv[i] === "execute" || argv[i] === "x") && i > 0 && (argv[i - 1] === "actions" || argv[i - 1] === "a")) {
|
|
2433
|
+
execIdx = i;
|
|
2434
|
+
break;
|
|
2435
|
+
}
|
|
2436
|
+
}
|
|
2437
|
+
if (execIdx === -1) {
|
|
2438
|
+
error('Could not locate "actions execute" in argv');
|
|
2439
|
+
}
|
|
2440
|
+
const raw = argv.slice(execIdx + 1);
|
|
2441
|
+
const flags = { dryRun: false, mock: false, skipValidation: false, maxConcurrency: 5 };
|
|
2442
|
+
const cleaned = [];
|
|
2443
|
+
for (let i = 0; i < raw.length; i++) {
|
|
2444
|
+
const t = raw[i];
|
|
2445
|
+
if (t === "--parallel" || t === "--agent") continue;
|
|
2446
|
+
if (t === "--dry-run") {
|
|
2447
|
+
flags.dryRun = true;
|
|
2448
|
+
continue;
|
|
2449
|
+
}
|
|
2450
|
+
if (t === "--mock") {
|
|
2451
|
+
flags.mock = true;
|
|
2452
|
+
continue;
|
|
2453
|
+
}
|
|
2454
|
+
if (t === "--skip-validation") {
|
|
2455
|
+
flags.skipValidation = true;
|
|
2456
|
+
continue;
|
|
2457
|
+
}
|
|
2458
|
+
if (t === "--max-concurrency") {
|
|
2459
|
+
flags.maxConcurrency = parseInt(raw[++i], 10) || 5;
|
|
2460
|
+
continue;
|
|
2461
|
+
}
|
|
2462
|
+
cleaned.push(t);
|
|
2463
|
+
}
|
|
2464
|
+
const segmentArrays = [];
|
|
2465
|
+
let current = [];
|
|
2466
|
+
for (const token of cleaned) {
|
|
2467
|
+
if (token === "--") {
|
|
2468
|
+
if (current.length > 0) segmentArrays.push(current);
|
|
2469
|
+
current = [];
|
|
2470
|
+
} else {
|
|
2471
|
+
current.push(token);
|
|
2472
|
+
}
|
|
2473
|
+
}
|
|
2474
|
+
if (current.length > 0) segmentArrays.push(current);
|
|
2475
|
+
if (segmentArrays.length === 0) {
|
|
2476
|
+
error("--parallel requires at least one action segment. Usage: --parallel <platform> <actionId> <connectionKey> [-d ...] [-- <next action> ...]");
|
|
2477
|
+
}
|
|
2478
|
+
const segments = segmentArrays.map((tokens, i) => parseSegmentTokens(tokens, i + 1));
|
|
2479
|
+
return { segments, flags };
|
|
2480
|
+
}
|
|
2481
|
+
function parseSegmentTokens(tokens, segmentIndex) {
|
|
2482
|
+
if (tokens.length < 3) {
|
|
2483
|
+
error(`Segment ${segmentIndex}: expected <platform> <actionId> <connectionKey>, got ${tokens.length} token(s): ${tokens.join(" ")}`);
|
|
2484
|
+
}
|
|
2485
|
+
const segment = {
|
|
2486
|
+
platform: tokens[0],
|
|
2487
|
+
actionId: tokens[1],
|
|
2488
|
+
connectionKey: tokens[2]
|
|
2489
|
+
};
|
|
2490
|
+
for (let i = 3; i < tokens.length; i++) {
|
|
2491
|
+
const t = tokens[i];
|
|
2492
|
+
if (t === "-d" || t === "--data") {
|
|
2493
|
+
segment.data = tokens[++i];
|
|
2494
|
+
continue;
|
|
2495
|
+
}
|
|
2496
|
+
if (t === "--path-vars") {
|
|
2497
|
+
segment.pathVars = tokens[++i];
|
|
2498
|
+
continue;
|
|
2499
|
+
}
|
|
2500
|
+
if (t === "--query-params") {
|
|
2501
|
+
segment.queryParams = tokens[++i];
|
|
2502
|
+
continue;
|
|
2503
|
+
}
|
|
2504
|
+
if (t === "--headers") {
|
|
2505
|
+
segment.headers = tokens[++i];
|
|
2506
|
+
continue;
|
|
2507
|
+
}
|
|
2508
|
+
if (t === "--form-data") {
|
|
2509
|
+
segment.formData = true;
|
|
2510
|
+
continue;
|
|
2511
|
+
}
|
|
2512
|
+
if (t === "--form-url-encoded") {
|
|
2513
|
+
segment.formUrlEncoded = true;
|
|
2514
|
+
continue;
|
|
2515
|
+
}
|
|
2516
|
+
error(`Segment ${segmentIndex}: unknown option "${t}"`);
|
|
2517
|
+
}
|
|
2518
|
+
return segment;
|
|
2519
|
+
}
|
|
2520
|
+
function tryParseJson(value, label) {
|
|
2521
|
+
try {
|
|
2522
|
+
return { value: JSON.parse(value) };
|
|
2523
|
+
} catch {
|
|
2524
|
+
return { error: `Invalid JSON for ${label}: ${value}` };
|
|
2525
|
+
}
|
|
2526
|
+
}
|
|
2527
|
+
async function actionsExecuteParallelCommand() {
|
|
2528
|
+
const { segments, flags } = parseParallelSegments();
|
|
2529
|
+
const { apiKey, permissions, actionIds, connectionKeys, knowledgeAgent } = getConfig();
|
|
2530
|
+
if (knowledgeAgent) {
|
|
2531
|
+
error("Action execution is disabled (knowledge-only mode).");
|
|
2532
|
+
}
|
|
2533
|
+
const api = new OneApi(apiKey, getApiBase());
|
|
2534
|
+
const prepared = [];
|
|
2535
|
+
const errors = [];
|
|
2536
|
+
for (let i = 0; i < segments.length; i++) {
|
|
2537
|
+
const seg = segments[i];
|
|
2538
|
+
const label = `${seg.platform}/${seg.actionId}`;
|
|
2539
|
+
const segErrors = [];
|
|
2540
|
+
if (!isActionAllowed(seg.actionId, actionIds)) {
|
|
2541
|
+
segErrors.push(`Action "${seg.actionId}" is not in the allowed action list`);
|
|
2542
|
+
}
|
|
2543
|
+
if (!connectionKeys.includes("*") && !connectionKeys.includes(seg.connectionKey)) {
|
|
2544
|
+
segErrors.push(`Connection key "${seg.connectionKey}" is not allowed`);
|
|
2545
|
+
}
|
|
2546
|
+
let actionDetails;
|
|
2547
|
+
try {
|
|
2548
|
+
actionDetails = await api.getActionDetails(seg.actionId);
|
|
2549
|
+
} catch (err) {
|
|
2550
|
+
segErrors.push(`Action not found: ${err instanceof Error ? err.message : String(err)}`);
|
|
2551
|
+
}
|
|
2552
|
+
if (actionDetails && !isMethodAllowed(actionDetails.method, permissions)) {
|
|
2553
|
+
segErrors.push(`Method "${actionDetails.method}" not allowed under "${permissions}" permission level`);
|
|
2554
|
+
}
|
|
2555
|
+
let data;
|
|
2556
|
+
let pathVariables;
|
|
2557
|
+
let queryParams;
|
|
2558
|
+
let headers;
|
|
2559
|
+
if (seg.data) {
|
|
2560
|
+
const r = tryParseJson(seg.data, `segment ${i + 1} --data`);
|
|
2561
|
+
if (r.error) segErrors.push(r.error);
|
|
2562
|
+
else data = r.value;
|
|
2563
|
+
}
|
|
2564
|
+
if (seg.pathVars) {
|
|
2565
|
+
const r = tryParseJson(seg.pathVars, `segment ${i + 1} --path-vars`);
|
|
2566
|
+
if (r.error) segErrors.push(r.error);
|
|
2567
|
+
else pathVariables = r.value;
|
|
2568
|
+
}
|
|
2569
|
+
if (seg.queryParams) {
|
|
2570
|
+
const r = tryParseJson(seg.queryParams, `segment ${i + 1} --query-params`);
|
|
2571
|
+
if (r.error) segErrors.push(r.error);
|
|
2572
|
+
else queryParams = r.value;
|
|
2573
|
+
}
|
|
2574
|
+
if (seg.headers) {
|
|
2575
|
+
const r = tryParseJson(seg.headers, `segment ${i + 1} --headers`);
|
|
2576
|
+
if (r.error) segErrors.push(r.error);
|
|
2577
|
+
else headers = r.value;
|
|
2578
|
+
}
|
|
2579
|
+
if (actionDetails && !flags.skipValidation && segErrors.length === 0) {
|
|
2580
|
+
const validation = validateActionInput(actionDetails, { data, pathVariables, queryParams });
|
|
2581
|
+
if (!validation.valid) {
|
|
2582
|
+
for (const m of validation.missing) {
|
|
2583
|
+
segErrors.push(`Missing ${m.flag} "${m.param}"${m.description ? ` \u2014 ${m.description}` : ""}`);
|
|
2584
|
+
}
|
|
2585
|
+
}
|
|
2586
|
+
}
|
|
2587
|
+
if (segErrors.length > 0) {
|
|
2588
|
+
errors.push({ segment: i + 1, label, messages: segErrors });
|
|
2589
|
+
} else if (actionDetails) {
|
|
2590
|
+
prepared.push({ segment: seg, index: i, actionDetails, data, pathVariables, queryParams, headers });
|
|
2591
|
+
}
|
|
2592
|
+
}
|
|
2593
|
+
if (errors.length > 0) {
|
|
2594
|
+
if (isAgentMode()) {
|
|
2595
|
+
json({ error: "Validation failed", segments: errors });
|
|
2596
|
+
process.exit(1);
|
|
2597
|
+
}
|
|
2598
|
+
console.log();
|
|
2599
|
+
for (const e of errors) {
|
|
2600
|
+
console.log(` ${pc6.red("\u2717")} Segment ${e.segment} (${e.label}):`);
|
|
2601
|
+
for (const msg of e.messages) {
|
|
2602
|
+
console.log(` ${msg}`);
|
|
2603
|
+
}
|
|
2604
|
+
}
|
|
2605
|
+
console.log();
|
|
2606
|
+
error(`Validation failed for ${errors.length} segment(s). Fix errors above before executing.`);
|
|
2607
|
+
}
|
|
2608
|
+
const total = prepared.length;
|
|
2609
|
+
const overallStart = Date.now();
|
|
2610
|
+
const results = [];
|
|
2611
|
+
for (let i = 0; i < total; i += flags.maxConcurrency) {
|
|
2612
|
+
const batch = prepared.slice(i, i + flags.maxConcurrency);
|
|
2613
|
+
const settled = await Promise.allSettled(
|
|
2614
|
+
batch.map(async (action) => {
|
|
2615
|
+
const start = Date.now();
|
|
2616
|
+
const seg = action.segment;
|
|
2617
|
+
const segIdx = action.index + 1;
|
|
2618
|
+
if (flags.mock) {
|
|
2619
|
+
const mockResponse = action.actionDetails.ioSchema?.ioExample?.output ?? null;
|
|
2620
|
+
return {
|
|
2621
|
+
segment: segIdx,
|
|
2622
|
+
platform: seg.platform,
|
|
2623
|
+
actionId: seg.actionId,
|
|
2624
|
+
status: "success",
|
|
2625
|
+
durationMs: Date.now() - start,
|
|
2626
|
+
mock: true,
|
|
2627
|
+
request: { method: action.actionDetails.method, url: action.actionDetails.path },
|
|
2628
|
+
response: mockResponse
|
|
2629
|
+
};
|
|
2630
|
+
}
|
|
2631
|
+
const result = await api.executePassthroughRequest({
|
|
2632
|
+
platform: seg.platform,
|
|
2633
|
+
actionId: seg.actionId,
|
|
2634
|
+
connectionKey: seg.connectionKey,
|
|
2635
|
+
data: action.data,
|
|
2636
|
+
pathVariables: action.pathVariables,
|
|
2637
|
+
queryParams: action.queryParams,
|
|
2638
|
+
headers: action.headers,
|
|
2639
|
+
isFormData: seg.formData,
|
|
2640
|
+
isFormUrlEncoded: seg.formUrlEncoded,
|
|
2641
|
+
dryRun: flags.dryRun
|
|
2642
|
+
}, action.actionDetails);
|
|
2643
|
+
return {
|
|
2644
|
+
segment: segIdx,
|
|
2645
|
+
platform: seg.platform,
|
|
2646
|
+
actionId: seg.actionId,
|
|
2647
|
+
status: "success",
|
|
2648
|
+
durationMs: Date.now() - start,
|
|
2649
|
+
dryRun: flags.dryRun || void 0,
|
|
2650
|
+
request: {
|
|
2651
|
+
method: result.requestConfig.method,
|
|
2652
|
+
url: result.requestConfig.url,
|
|
2653
|
+
...flags.dryRun ? { headers: result.requestConfig.headers, data: result.requestConfig.data } : {}
|
|
2654
|
+
},
|
|
2655
|
+
response: flags.dryRun ? void 0 : result.responseData
|
|
2656
|
+
};
|
|
2657
|
+
})
|
|
2658
|
+
);
|
|
2659
|
+
for (const s of settled) {
|
|
2660
|
+
if (s.status === "fulfilled") {
|
|
2661
|
+
results.push(s.value);
|
|
2662
|
+
} else {
|
|
2663
|
+
const batchIdx = settled.indexOf(s);
|
|
2664
|
+
const action = batch[batchIdx];
|
|
2665
|
+
results.push({
|
|
2666
|
+
segment: action.index + 1,
|
|
2667
|
+
platform: action.segment.platform,
|
|
2668
|
+
actionId: action.segment.actionId,
|
|
2669
|
+
status: "error",
|
|
2670
|
+
durationMs: 0,
|
|
2671
|
+
error: s.reason instanceof Error ? s.reason.message : String(s.reason)
|
|
2672
|
+
});
|
|
2673
|
+
}
|
|
2674
|
+
}
|
|
2675
|
+
}
|
|
2676
|
+
const totalDurationMs = Date.now() - overallStart;
|
|
2677
|
+
const succeeded = results.filter((r) => r.status === "success").length;
|
|
2678
|
+
const failed = results.filter((r) => r.status === "error").length;
|
|
2679
|
+
if (isAgentMode()) {
|
|
2680
|
+
json({
|
|
2681
|
+
parallel: true,
|
|
2682
|
+
totalDurationMs,
|
|
2683
|
+
succeeded,
|
|
2684
|
+
failed,
|
|
2685
|
+
results
|
|
2686
|
+
});
|
|
2687
|
+
if (failed > 0) process.exit(1);
|
|
2688
|
+
return;
|
|
2689
|
+
}
|
|
2690
|
+
console.log();
|
|
2691
|
+
for (const r of results) {
|
|
2692
|
+
const label = `${r.platform}/${r.actionId}`;
|
|
2693
|
+
const time = pc6.dim(`(${(r.durationMs / 1e3).toFixed(2)}s)`);
|
|
2694
|
+
if (r.mock) {
|
|
2695
|
+
console.log(` [${r.segment}/${total}] ${label} ${pc6.cyan("\u25C7 mock")} ${time}`);
|
|
2696
|
+
if (r.response) console.log(` ${pc6.dim(JSON.stringify(r.response, null, 2).split("\n").join("\n "))}`);
|
|
2697
|
+
} else if (r.dryRun) {
|
|
2698
|
+
console.log(` [${r.segment}/${total}] ${label} ${pc6.yellow("\u2298 dry-run")} ${time}`);
|
|
2699
|
+
} else if (r.status === "success") {
|
|
2700
|
+
console.log(` [${r.segment}/${total}] ${label} ${pc6.green("\u2713")} ${time}`);
|
|
2701
|
+
if (r.response) console.log(` ${pc6.dim(JSON.stringify(r.response, null, 2).split("\n").join("\n "))}`);
|
|
2702
|
+
} else {
|
|
2703
|
+
console.log(` [${r.segment}/${total}] ${label} ${pc6.red("\u2717")} ${time} \u2014 ${r.error}`);
|
|
2704
|
+
}
|
|
2705
|
+
}
|
|
2706
|
+
console.log();
|
|
2707
|
+
const tag = flags.mock ? "mock" : flags.dryRun ? "dry-run" : void 0;
|
|
2708
|
+
if (tag) {
|
|
2709
|
+
console.log(` Summary: ${total} ${tag} (${(totalDurationMs / 1e3).toFixed(2)}s total)`);
|
|
2710
|
+
} else {
|
|
2711
|
+
console.log(` Summary: ${succeeded} succeeded, ${failed} failed (${(totalDurationMs / 1e3).toFixed(2)}s total)`);
|
|
2712
|
+
}
|
|
2713
|
+
console.log();
|
|
2714
|
+
}
|
|
2377
2715
|
function colorMethod(method) {
|
|
2378
2716
|
switch (method.toUpperCase()) {
|
|
2379
2717
|
case "GET":
|
|
@@ -2461,32 +2799,32 @@ function validateStepsArray(steps, pathPrefix, errors) {
|
|
|
2461
2799
|
const validTypes = FLOW_SCHEMA.stepTypes.map((st) => st.type);
|
|
2462
2800
|
for (let i = 0; i < steps.length; i++) {
|
|
2463
2801
|
const step = steps[i];
|
|
2464
|
-
const
|
|
2802
|
+
const path16 = `${pathPrefix}[${i}]`;
|
|
2465
2803
|
if (!step || typeof step !== "object" || Array.isArray(step)) {
|
|
2466
|
-
errors.push({ path:
|
|
2804
|
+
errors.push({ path: path16, message: "Step must be an object" });
|
|
2467
2805
|
continue;
|
|
2468
2806
|
}
|
|
2469
2807
|
const s = step;
|
|
2470
2808
|
if (!s.id || typeof s.id !== "string") {
|
|
2471
|
-
errors.push({ path: `${
|
|
2809
|
+
errors.push({ path: `${path16}.id`, message: 'Step must have a string "id"' });
|
|
2472
2810
|
}
|
|
2473
2811
|
if (!s.name || typeof s.name !== "string") {
|
|
2474
|
-
errors.push({ path: `${
|
|
2812
|
+
errors.push({ path: `${path16}.name`, message: 'Step must have a string "name"' });
|
|
2475
2813
|
}
|
|
2476
2814
|
if (!s.type || !validTypes.includes(s.type)) {
|
|
2477
|
-
errors.push({ path: `${
|
|
2815
|
+
errors.push({ path: `${path16}.type`, message: `Step type must be one of: ${validTypes.join(", ")}` });
|
|
2478
2816
|
continue;
|
|
2479
2817
|
}
|
|
2480
2818
|
if (s.requires !== void 0) {
|
|
2481
2819
|
if (!Array.isArray(s.requires)) {
|
|
2482
|
-
errors.push({ path: `${
|
|
2820
|
+
errors.push({ path: `${path16}.requires`, message: '"requires" must be an array of selector strings (e.g. ["$.steps.foo.output.bar"])' });
|
|
2483
2821
|
} else {
|
|
2484
2822
|
for (let r = 0; r < s.requires.length; r++) {
|
|
2485
2823
|
const sel = s.requires[r];
|
|
2486
2824
|
if (typeof sel !== "string") {
|
|
2487
|
-
errors.push({ path: `${
|
|
2825
|
+
errors.push({ path: `${path16}.requires[${r}]`, message: '"requires" entry must be a selector string' });
|
|
2488
2826
|
} else if (!sel.startsWith("$.")) {
|
|
2489
|
-
errors.push({ path: `${
|
|
2827
|
+
errors.push({ path: `${path16}.requires[${r}]`, message: `"requires" entry "${sel}" must be a selector starting with "$." (e.g. "$.steps.foo.output.bar")` });
|
|
2490
2828
|
}
|
|
2491
2829
|
}
|
|
2492
2830
|
}
|
|
@@ -2494,7 +2832,7 @@ function validateStepsArray(steps, pathPrefix, errors) {
|
|
|
2494
2832
|
if (s.onError && typeof s.onError === "object") {
|
|
2495
2833
|
const oe = s.onError;
|
|
2496
2834
|
if (!FLOW_SCHEMA.errorStrategies.includes(oe.strategy)) {
|
|
2497
|
-
errors.push({ path: `${
|
|
2835
|
+
errors.push({ path: `${path16}.onError.strategy`, message: `Error strategy must be one of: ${FLOW_SCHEMA.errorStrategies.join(", ")}` });
|
|
2498
2836
|
}
|
|
2499
2837
|
}
|
|
2500
2838
|
const descriptor = getStepTypeDescriptor(s.type);
|
|
@@ -2504,14 +2842,14 @@ function validateStepsArray(steps, pathPrefix, errors) {
|
|
|
2504
2842
|
if (!configObj || typeof configObj !== "object") {
|
|
2505
2843
|
const hint = detectFlatConfigHint(s, descriptor);
|
|
2506
2844
|
errors.push({
|
|
2507
|
-
path: `${
|
|
2845
|
+
path: `${path16}.${configKey}`,
|
|
2508
2846
|
message: `${capitalize(descriptor.type)} step must have a "${configKey}" config object${hint}`
|
|
2509
2847
|
});
|
|
2510
2848
|
continue;
|
|
2511
2849
|
}
|
|
2512
2850
|
const config2 = configObj;
|
|
2513
2851
|
for (const [fieldName, fd] of Object.entries(descriptor.fields)) {
|
|
2514
|
-
const fieldPath = `${
|
|
2852
|
+
const fieldPath = `${path16}.${configKey}.${fieldName}`;
|
|
2515
2853
|
const value = config2[fieldName];
|
|
2516
2854
|
if (fd.required && (value === void 0 || value === null || value === "")) {
|
|
2517
2855
|
errors.push({ path: fieldPath, message: `${capitalize(descriptor.type)} must have ${fd.type === "string" ? "a string" : fd.type === "array" ? "a" : "a"} "${fieldName}"` });
|
|
@@ -2547,24 +2885,24 @@ function validateStepsArray(steps, pathPrefix, errors) {
|
|
|
2547
2885
|
const hasSource = typeof config2.source === "string" && config2.source.length > 0;
|
|
2548
2886
|
const hasModule = typeof config2.module === "string" && config2.module.length > 0;
|
|
2549
2887
|
if (!hasSource && !hasModule) {
|
|
2550
|
-
errors.push({ path: `${
|
|
2888
|
+
errors.push({ path: `${path16}.${configKey}`, message: 'Code step must define either "source" (inline JS) or "module" (path to .mjs file)' });
|
|
2551
2889
|
} else if (hasSource && hasModule) {
|
|
2552
|
-
errors.push({ path: `${
|
|
2890
|
+
errors.push({ path: `${path16}.${configKey}`, message: 'Code step cannot define both "source" and "module" \u2014 pick one' });
|
|
2553
2891
|
}
|
|
2554
2892
|
if (hasModule) {
|
|
2555
2893
|
const m = config2.module;
|
|
2556
2894
|
if (m.startsWith("/") || /^[a-zA-Z]:[\\/]/.test(m)) {
|
|
2557
|
-
errors.push({ path: `${
|
|
2895
|
+
errors.push({ path: `${path16}.${configKey}.module`, message: "Code module path must be relative to the flow folder (no absolute paths)" });
|
|
2558
2896
|
} else if (m.split(/[\\/]/).includes("..")) {
|
|
2559
|
-
errors.push({ path: `${
|
|
2897
|
+
errors.push({ path: `${path16}.${configKey}.module`, message: 'Code module path must not escape the flow folder ("..")' });
|
|
2560
2898
|
} else if (!m.endsWith(".mjs")) {
|
|
2561
|
-
errors.push({ path: `${
|
|
2899
|
+
errors.push({ path: `${path16}.${configKey}.module`, message: "Code module must be a .mjs file" });
|
|
2562
2900
|
}
|
|
2563
2901
|
}
|
|
2564
2902
|
if (hasSource) {
|
|
2565
2903
|
const syntaxError = checkCodeSourceSyntax(config2.source);
|
|
2566
2904
|
if (syntaxError) {
|
|
2567
|
-
errors.push({ path: `${
|
|
2905
|
+
errors.push({ path: `${path16}.${configKey}.source`, message: `Syntax error in code step: ${syntaxError}` });
|
|
2568
2906
|
}
|
|
2569
2907
|
}
|
|
2570
2908
|
}
|
|
@@ -2599,16 +2937,16 @@ function validateStepIds(flow2) {
|
|
|
2599
2937
|
function collectIds(steps, pathPrefix) {
|
|
2600
2938
|
for (let i = 0; i < steps.length; i++) {
|
|
2601
2939
|
const step = steps[i];
|
|
2602
|
-
const
|
|
2940
|
+
const path16 = `${pathPrefix}[${i}]`;
|
|
2603
2941
|
if (seen.has(step.id)) {
|
|
2604
|
-
errors.push({ path: `${
|
|
2942
|
+
errors.push({ path: `${path16}.id`, message: `Duplicate step ID: "${step.id}"` });
|
|
2605
2943
|
} else {
|
|
2606
2944
|
seen.add(step.id);
|
|
2607
2945
|
}
|
|
2608
2946
|
for (const { configKey, fieldName } of nestedKeys) {
|
|
2609
2947
|
const config2 = step[configKey];
|
|
2610
2948
|
if (config2 && Array.isArray(config2[fieldName])) {
|
|
2611
|
-
collectIds(config2[fieldName], `${
|
|
2949
|
+
collectIds(config2[fieldName], `${path16}.${configKey}.${fieldName}`);
|
|
2612
2950
|
}
|
|
2613
2951
|
}
|
|
2614
2952
|
}
|
|
@@ -2656,7 +2994,7 @@ function validateSelectorReferences(flow2) {
|
|
|
2656
2994
|
}
|
|
2657
2995
|
return selectors;
|
|
2658
2996
|
}
|
|
2659
|
-
function checkSelectors(selectors,
|
|
2997
|
+
function checkSelectors(selectors, path16, precedingStepIds) {
|
|
2660
2998
|
for (const selector of selectors) {
|
|
2661
2999
|
const parts = selector.split(".");
|
|
2662
3000
|
if (parts.length < 3) continue;
|
|
@@ -2664,15 +3002,15 @@ function validateSelectorReferences(flow2) {
|
|
|
2664
3002
|
if (root === "input") {
|
|
2665
3003
|
const inputName = parts[2];
|
|
2666
3004
|
if (!inputNames.has(inputName)) {
|
|
2667
|
-
errors.push({ path:
|
|
3005
|
+
errors.push({ path: path16, message: `Selector "${selector}" references undefined input "${inputName}"` });
|
|
2668
3006
|
}
|
|
2669
3007
|
} else if (root === "steps") {
|
|
2670
3008
|
const stepId = parts[2].replace(/[\[\]]/g, "").split(/[\[\]]/)[0];
|
|
2671
3009
|
if (!allStepIds.has(stepId)) {
|
|
2672
|
-
errors.push({ path:
|
|
3010
|
+
errors.push({ path: path16, message: `Selector "${selector}" references undefined step "${stepId}"` });
|
|
2673
3011
|
} else if (precedingStepIds && !precedingStepIds.has(stepId)) {
|
|
2674
3012
|
errors.push({
|
|
2675
|
-
path:
|
|
3013
|
+
path: path16,
|
|
2676
3014
|
message: `Selector "${selector}" references step "${stepId}" which is declared after the current step. Steps execute in declaration order, so this will always resolve to undefined at runtime \u2014 move the dependency earlier in the steps array.`
|
|
2677
3015
|
});
|
|
2678
3016
|
}
|
|
@@ -2680,20 +3018,20 @@ function validateSelectorReferences(flow2) {
|
|
|
2680
3018
|
}
|
|
2681
3019
|
}
|
|
2682
3020
|
const EXPRESSION_FIELDS = /* @__PURE__ */ new Set(["condition.expression", "while.condition"]);
|
|
2683
|
-
function checkOperatorsInSelectorField(value,
|
|
3021
|
+
function checkOperatorsInSelectorField(value, path16) {
|
|
2684
3022
|
if (typeof value === "string" && value.startsWith("$.")) {
|
|
2685
3023
|
if (value.includes("||")) {
|
|
2686
|
-
errors.push({ path:
|
|
3024
|
+
errors.push({ path: path16, message: `Selector "${value}" contains unsupported operator "||". Selectors in data fields use dot-path resolution, not JS evaluation. Use the "default" field on the input definition instead, or use a "code" step for complex expressions.` });
|
|
2687
3025
|
} else if (value.includes("&&")) {
|
|
2688
|
-
errors.push({ path:
|
|
3026
|
+
errors.push({ path: path16, message: `Selector "${value}" contains unsupported operator "&&". Selectors in data fields use dot-path resolution, not JS evaluation. Use a "condition" step or "code" step for complex expressions.` });
|
|
2689
3027
|
}
|
|
2690
3028
|
} else if (value && typeof value === "object" && !Array.isArray(value)) {
|
|
2691
3029
|
for (const [k, v] of Object.entries(value)) {
|
|
2692
|
-
checkOperatorsInSelectorField(v, `${
|
|
3030
|
+
checkOperatorsInSelectorField(v, `${path16}.${k}`);
|
|
2693
3031
|
}
|
|
2694
3032
|
} else if (Array.isArray(value)) {
|
|
2695
3033
|
for (let i = 0; i < value.length; i++) {
|
|
2696
|
-
checkOperatorsInSelectorField(value[i], `${
|
|
3034
|
+
checkOperatorsInSelectorField(value[i], `${path16}[${i}]`);
|
|
2697
3035
|
}
|
|
2698
3036
|
}
|
|
2699
3037
|
}
|
|
@@ -2761,10 +3099,10 @@ var VALID_OUTPUT_SCHEMA_TYPES = /* @__PURE__ */ new Set(["string", "number", "bo
|
|
|
2761
3099
|
function isOutputSchemaObject(v) {
|
|
2762
3100
|
return !!v && typeof v === "object" && !Array.isArray(v);
|
|
2763
3101
|
}
|
|
2764
|
-
function walkOutputSchema(schema,
|
|
3102
|
+
function walkOutputSchema(schema, path16) {
|
|
2765
3103
|
let current = schema;
|
|
2766
|
-
for (let i = 0; i <
|
|
2767
|
-
const seg =
|
|
3104
|
+
for (let i = 0; i < path16.length; i++) {
|
|
3105
|
+
const seg = path16[i];
|
|
2768
3106
|
if (typeof current === "string") {
|
|
2769
3107
|
return current === "unknown" || current === "object" || current === "array" ? "opaque" : "opaque";
|
|
2770
3108
|
}
|
|
@@ -3132,6 +3470,7 @@ ${pc7.yellow("Pausing after current step completes...")} (run ID: ${runId})`);
|
|
|
3132
3470
|
mock: options.mock,
|
|
3133
3471
|
verbose: options.verbose,
|
|
3134
3472
|
allowBash: options.allowBash,
|
|
3473
|
+
skipValidation: options.skipValidation,
|
|
3135
3474
|
rootDir,
|
|
3136
3475
|
onEvent
|
|
3137
3476
|
});
|
|
@@ -3900,138 +4239,3326 @@ async function relayEventTypesCommand(platform) {
|
|
|
3900
4239
|
}
|
|
3901
4240
|
}
|
|
3902
4241
|
|
|
3903
|
-
// src/
|
|
3904
|
-
|
|
3905
|
-
|
|
3906
|
-
if (
|
|
3907
|
-
|
|
3908
|
-
|
|
3909
|
-
|
|
3910
|
-
|
|
3911
|
-
|
|
3912
|
-
|
|
3913
|
-
|
|
3914
|
-
|
|
3915
|
-
|
|
3916
|
-
|
|
3917
|
-
|
|
3918
|
-
|
|
3919
|
-
if (
|
|
3920
|
-
|
|
3921
|
-
|
|
3922
|
-
|
|
3923
|
-
|
|
3924
|
-
|
|
3925
|
-
|
|
3926
|
-
|
|
3927
|
-
|
|
3928
|
-
|
|
3929
|
-
|
|
3930
|
-
|
|
3931
|
-
|
|
3932
|
-
type: e.type,
|
|
3933
|
-
key: e.entry.key,
|
|
3934
|
-
cachedAt: e.entry.cachedAt,
|
|
3935
|
-
age: formatAge(getAge(e.entry)),
|
|
3936
|
-
ttl: e.entry.ttl,
|
|
3937
|
-
fresh: isFresh(e.entry),
|
|
3938
|
-
etag: e.entry.etag,
|
|
3939
|
-
path: e.filePath
|
|
3940
|
-
}))
|
|
4242
|
+
// src/lib/sync/models.ts
|
|
4243
|
+
function parseActionType(actionKey) {
|
|
4244
|
+
const parts = actionKey.split("::");
|
|
4245
|
+
if (parts.length < 5) return null;
|
|
4246
|
+
return parts[4];
|
|
4247
|
+
}
|
|
4248
|
+
async function discoverModels(api, platform) {
|
|
4249
|
+
const actions2 = await api.listAvailableActions(platform);
|
|
4250
|
+
const listActionTypes = /* @__PURE__ */ new Set(["get_many", "list", "get_all"]);
|
|
4251
|
+
const modelMap = /* @__PURE__ */ new Map();
|
|
4252
|
+
for (const action of actions2) {
|
|
4253
|
+
const actionType = parseActionType(action.key);
|
|
4254
|
+
if (!actionType) continue;
|
|
4255
|
+
if (!listActionTypes.has(actionType)) continue;
|
|
4256
|
+
const modelName = action.modelName;
|
|
4257
|
+
if (!modelName) continue;
|
|
4258
|
+
if (modelMap.has(modelName)) {
|
|
4259
|
+
const existing = modelMap.get(modelName);
|
|
4260
|
+
const existingType = parseActionType(existing.listAction.actionId);
|
|
4261
|
+
if (existingType === "get_many") continue;
|
|
4262
|
+
}
|
|
4263
|
+
modelMap.set(modelName, {
|
|
4264
|
+
name: modelName,
|
|
4265
|
+
displayName: action.title,
|
|
4266
|
+
listAction: {
|
|
4267
|
+
actionId: action.key,
|
|
4268
|
+
path: action.path,
|
|
4269
|
+
method: action.method
|
|
4270
|
+
}
|
|
3941
4271
|
});
|
|
3942
|
-
return;
|
|
3943
|
-
}
|
|
3944
|
-
if (filtered.length === 0) {
|
|
3945
|
-
console.log(options.expired ? "No expired cache entries" : "No cached entries");
|
|
3946
|
-
return;
|
|
3947
4272
|
}
|
|
3948
|
-
const
|
|
3949
|
-
|
|
3950
|
-
|
|
3951
|
-
|
|
3952
|
-
|
|
3953
|
-
|
|
3954
|
-
|
|
3955
|
-
|
|
3956
|
-
|
|
3957
|
-
|
|
3958
|
-
|
|
3959
|
-
|
|
3960
|
-
|
|
3961
|
-
|
|
4273
|
+
const models = Array.from(modelMap.values());
|
|
4274
|
+
await Promise.all(
|
|
4275
|
+
models.map(async (model) => {
|
|
4276
|
+
try {
|
|
4277
|
+
const searchResults = await api.searchActions(platform, model.displayName, "execute");
|
|
4278
|
+
const resolved = searchResults.find(
|
|
4279
|
+
(a) => a.path === model.listAction.path && a.method === model.listAction.method
|
|
4280
|
+
);
|
|
4281
|
+
if (resolved?.systemId) {
|
|
4282
|
+
model.listAction.actionId = resolved.systemId;
|
|
4283
|
+
}
|
|
4284
|
+
} catch {
|
|
4285
|
+
}
|
|
4286
|
+
})
|
|
3962
4287
|
);
|
|
4288
|
+
return models.sort((a, b) => a.name.localeCompare(b.name));
|
|
3963
4289
|
}
|
|
3964
|
-
|
|
3965
|
-
|
|
3966
|
-
|
|
3967
|
-
|
|
4290
|
+
|
|
4291
|
+
// src/lib/sync/profile.ts
|
|
4292
|
+
import fs8 from "fs";
|
|
4293
|
+
import path8 from "path";
|
|
4294
|
+
var PROFILES_DIR = path8.join(".one", "sync", "profiles");
|
|
4295
|
+
function profilePath(platform, model) {
|
|
4296
|
+
return path8.join(PROFILES_DIR, `${platform}_${model}.json`);
|
|
4297
|
+
}
|
|
4298
|
+
function readProfile(platform, model) {
|
|
4299
|
+
const filePath = profilePath(platform, model);
|
|
4300
|
+
try {
|
|
4301
|
+
if (!fs8.existsSync(filePath)) return null;
|
|
4302
|
+
const raw = fs8.readFileSync(filePath, "utf-8");
|
|
4303
|
+
return JSON.parse(raw);
|
|
4304
|
+
} catch {
|
|
4305
|
+
return null;
|
|
3968
4306
|
}
|
|
3969
|
-
|
|
3970
|
-
|
|
3971
|
-
|
|
3972
|
-
|
|
3973
|
-
|
|
3974
|
-
|
|
4307
|
+
}
|
|
4308
|
+
function writeProfile(profile) {
|
|
4309
|
+
const required = ["platform", "model", "connectionKey", "actionId", "resultsPath", "idField", "pagination"];
|
|
4310
|
+
for (const field of required) {
|
|
4311
|
+
if (!profile[field]) {
|
|
4312
|
+
throw new Error(`Missing required field: ${field}`);
|
|
3975
4313
|
}
|
|
3976
|
-
console.log("No cached entries to update");
|
|
3977
|
-
return;
|
|
3978
4314
|
}
|
|
3979
|
-
|
|
3980
|
-
|
|
3981
|
-
|
|
3982
|
-
|
|
3983
|
-
const
|
|
3984
|
-
|
|
4315
|
+
if (!profile.pagination.type) {
|
|
4316
|
+
throw new Error("Missing required field: pagination.type");
|
|
4317
|
+
}
|
|
4318
|
+
fs8.mkdirSync(PROFILES_DIR, { recursive: true });
|
|
4319
|
+
const filePath = profilePath(profile.platform, profile.model);
|
|
4320
|
+
fs8.writeFileSync(filePath, JSON.stringify(profile, null, 2));
|
|
4321
|
+
}
|
|
4322
|
+
function writeDraftProfile(platform, model, draft) {
|
|
4323
|
+
fs8.mkdirSync(PROFILES_DIR, { recursive: true });
|
|
4324
|
+
const filePath = profilePath(platform, model);
|
|
4325
|
+
fs8.writeFileSync(filePath, JSON.stringify(draft, null, 2));
|
|
4326
|
+
}
|
|
4327
|
+
function listProfiles(platform) {
|
|
4328
|
+
if (!fs8.existsSync(PROFILES_DIR)) return [];
|
|
4329
|
+
const files = fs8.readdirSync(PROFILES_DIR).filter((f) => f.endsWith(".json"));
|
|
4330
|
+
const profiles = [];
|
|
4331
|
+
for (const file of files) {
|
|
3985
4332
|
try {
|
|
3986
|
-
|
|
3987
|
-
|
|
3988
|
-
|
|
3989
|
-
|
|
3990
|
-
updated++;
|
|
3991
|
-
} else {
|
|
3992
|
-
const refreshed = { ...e.entry, cachedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
3993
|
-
writeCache2(e.filePath, refreshed);
|
|
3994
|
-
updated++;
|
|
4333
|
+
const raw = fs8.readFileSync(path8.join(PROFILES_DIR, file), "utf-8");
|
|
4334
|
+
const profile = JSON.parse(raw);
|
|
4335
|
+
if (!platform || profile.platform === platform) {
|
|
4336
|
+
profiles.push(profile);
|
|
3995
4337
|
}
|
|
3996
|
-
} catch
|
|
3997
|
-
failed++;
|
|
3998
|
-
errors.push({
|
|
3999
|
-
key: e.entry.key,
|
|
4000
|
-
error: err instanceof Error ? err.message : "Unknown error"
|
|
4001
|
-
});
|
|
4338
|
+
} catch {
|
|
4002
4339
|
}
|
|
4003
4340
|
}
|
|
4004
|
-
|
|
4005
|
-
|
|
4006
|
-
|
|
4007
|
-
|
|
4341
|
+
return profiles;
|
|
4342
|
+
}
|
|
4343
|
+
function generateTemplate(platform, model, actionId) {
|
|
4344
|
+
return {
|
|
4345
|
+
platform,
|
|
4346
|
+
model,
|
|
4347
|
+
connectionKey: "FILL_IN",
|
|
4348
|
+
actionId: actionId ?? "FILL_IN",
|
|
4349
|
+
resultsPath: "FILL_IN",
|
|
4350
|
+
idField: "FILL_IN",
|
|
4351
|
+
pagination: {
|
|
4352
|
+
type: "FILL_IN (cursor | token | offset | id | link | none)",
|
|
4353
|
+
nextPath: "FILL_IN",
|
|
4354
|
+
passAs: "FILL_IN (query:name | body:name | header:name)"
|
|
4355
|
+
}
|
|
4356
|
+
// Optional. Set to "body" for POST-body list endpoints (e.g. Notion /v1/search).
|
|
4357
|
+
// limitLocation: "query",
|
|
4358
|
+
// Optional. Page size param name. Set to "" to disable sending any page size.
|
|
4359
|
+
// limitParam: "limit",
|
|
4360
|
+
// Optional. Default page size (100).
|
|
4361
|
+
// defaultLimit: 100,
|
|
4362
|
+
};
|
|
4363
|
+
}
|
|
4364
|
+
|
|
4365
|
+
// src/lib/sync/pagination.ts
|
|
4366
|
+
function parsePassAs(passAs) {
|
|
4367
|
+
const colonIdx = passAs.indexOf(":");
|
|
4368
|
+
if (colonIdx === -1) {
|
|
4369
|
+
return { location: "query", paramName: passAs };
|
|
4370
|
+
}
|
|
4371
|
+
const location = passAs.slice(0, colonIdx);
|
|
4372
|
+
const paramName = passAs.slice(colonIdx + 1);
|
|
4373
|
+
if (location === "header") return { location: "header", paramName };
|
|
4374
|
+
if (location === "body") return { location: "body", paramName };
|
|
4375
|
+
return { location: "query", paramName };
|
|
4376
|
+
}
|
|
4377
|
+
function getNextPageParams(response, config2, currentPage, pageSize, records) {
|
|
4378
|
+
switch (config2.type) {
|
|
4379
|
+
case "cursor":
|
|
4380
|
+
case "token":
|
|
4381
|
+
case "link":
|
|
4382
|
+
return handleCursorLike(response, config2);
|
|
4383
|
+
case "offset":
|
|
4384
|
+
return handleOffset(response, config2, currentPage, pageSize, records);
|
|
4385
|
+
case "id":
|
|
4386
|
+
return handleId(response, config2, records);
|
|
4387
|
+
case "none":
|
|
4388
|
+
return null;
|
|
4389
|
+
default:
|
|
4390
|
+
return null;
|
|
4391
|
+
}
|
|
4392
|
+
}
|
|
4393
|
+
function buildParam(location, paramName, value) {
|
|
4394
|
+
if (location === "header") return { headers: { [paramName]: String(value) } };
|
|
4395
|
+
if (location === "body") return { bodyParams: { [paramName]: value } };
|
|
4396
|
+
return { queryParams: { [paramName]: value } };
|
|
4397
|
+
}
|
|
4398
|
+
function handleCursorLike(response, config2) {
|
|
4399
|
+
if (!config2.nextPath || !config2.passAs) return null;
|
|
4400
|
+
const nextValue = getByDotPath(response, config2.nextPath);
|
|
4401
|
+
if (nextValue === null || nextValue === void 0 || nextValue === "") return null;
|
|
4402
|
+
const { location, paramName } = parsePassAs(config2.passAs);
|
|
4403
|
+
return buildParam(location, paramName, nextValue);
|
|
4404
|
+
}
|
|
4405
|
+
function handleOffset(response, config2, currentPage, pageSize, records) {
|
|
4406
|
+
if (records.length === 0) return null;
|
|
4407
|
+
const nextOffset = (currentPage + 1) * pageSize;
|
|
4408
|
+
if (config2.totalPath) {
|
|
4409
|
+
const total = getByDotPath(response, config2.totalPath);
|
|
4410
|
+
if (typeof total === "number" && nextOffset >= total) return null;
|
|
4411
|
+
}
|
|
4412
|
+
if (!config2.passAs) return null;
|
|
4413
|
+
const { location, paramName } = parsePassAs(config2.passAs);
|
|
4414
|
+
return buildParam(location, paramName, nextOffset);
|
|
4415
|
+
}
|
|
4416
|
+
function handleId(response, config2, records) {
|
|
4417
|
+
if (records.length === 0) return null;
|
|
4418
|
+
if (config2.hasMorePath) {
|
|
4419
|
+
const hasMore = getByDotPath(response, config2.hasMorePath);
|
|
4420
|
+
if (hasMore === false) return null;
|
|
4421
|
+
}
|
|
4422
|
+
const lastRecord = records[records.length - 1];
|
|
4423
|
+
const idFieldName = config2.idField || "id";
|
|
4424
|
+
const lastId = typeof lastRecord === "object" && lastRecord !== null ? lastRecord[idFieldName] : null;
|
|
4425
|
+
if (lastId === null || lastId === void 0) return null;
|
|
4426
|
+
if (!config2.passAs) return null;
|
|
4427
|
+
const { location, paramName } = parsePassAs(config2.passAs);
|
|
4428
|
+
return buildParam(location, paramName, lastId);
|
|
4429
|
+
}
|
|
4430
|
+
|
|
4431
|
+
// src/lib/sync/state.ts
|
|
4432
|
+
import fs9 from "fs";
|
|
4433
|
+
import path9 from "path";
|
|
4434
|
+
var SYNC_DIR = path9.join(".one", "sync");
|
|
4435
|
+
var STATE_FILE = path9.join(SYNC_DIR, "sync_state.json");
|
|
4436
|
+
function readSyncState() {
|
|
4437
|
+
try {
|
|
4438
|
+
if (!fs9.existsSync(STATE_FILE)) return {};
|
|
4439
|
+
const raw = fs9.readFileSync(STATE_FILE, "utf-8");
|
|
4440
|
+
return JSON.parse(raw);
|
|
4441
|
+
} catch {
|
|
4442
|
+
return {};
|
|
4008
4443
|
}
|
|
4009
|
-
|
|
4010
|
-
|
|
4011
|
-
|
|
4012
|
-
|
|
4444
|
+
}
|
|
4445
|
+
function writeSyncState(state) {
|
|
4446
|
+
fs9.mkdirSync(SYNC_DIR, { recursive: true });
|
|
4447
|
+
const tmp = STATE_FILE + ".tmp";
|
|
4448
|
+
fs9.writeFileSync(tmp, JSON.stringify(state, null, 2));
|
|
4449
|
+
fs9.renameSync(tmp, STATE_FILE);
|
|
4450
|
+
}
|
|
4451
|
+
function getModelState(platform, model) {
|
|
4452
|
+
const state = readSyncState();
|
|
4453
|
+
return state[platform]?.[model] ?? null;
|
|
4454
|
+
}
|
|
4455
|
+
function updateModelState(platform, model, partial) {
|
|
4456
|
+
const state = readSyncState();
|
|
4457
|
+
if (!state[platform]) state[platform] = {};
|
|
4458
|
+
const existing = state[platform][model] ?? {
|
|
4459
|
+
lastSync: null,
|
|
4460
|
+
lastCursor: null,
|
|
4461
|
+
totalRecords: 0,
|
|
4462
|
+
pagesProcessed: 0,
|
|
4463
|
+
since: null,
|
|
4464
|
+
status: "idle"
|
|
4465
|
+
};
|
|
4466
|
+
state[platform][model] = { ...existing, ...partial };
|
|
4467
|
+
writeSyncState(state);
|
|
4468
|
+
}
|
|
4469
|
+
function removeModelState(platform, model) {
|
|
4470
|
+
const state = readSyncState();
|
|
4471
|
+
if (!state[platform]) return;
|
|
4472
|
+
if (model) {
|
|
4473
|
+
delete state[platform][model];
|
|
4474
|
+
if (Object.keys(state[platform]).length === 0) {
|
|
4475
|
+
delete state[platform];
|
|
4013
4476
|
}
|
|
4477
|
+
} else {
|
|
4478
|
+
delete state[platform];
|
|
4014
4479
|
}
|
|
4480
|
+
writeSyncState(state);
|
|
4015
4481
|
}
|
|
4016
4482
|
|
|
4017
|
-
// src/
|
|
4018
|
-
import
|
|
4019
|
-
|
|
4020
|
-
// src/lib/guide-content.ts
|
|
4021
|
-
var GUIDE_OVERVIEW = `# One CLI \u2014 Agent Guide
|
|
4022
|
-
|
|
4023
|
-
## Setup
|
|
4483
|
+
// src/lib/sync/db.ts
|
|
4484
|
+
import fs10 from "fs";
|
|
4485
|
+
import path10 from "path";
|
|
4024
4486
|
|
|
4025
|
-
|
|
4026
|
-
|
|
4027
|
-
|
|
4487
|
+
// src/lib/sync/sqlite-loader.ts
|
|
4488
|
+
var cached = null;
|
|
4489
|
+
async function loadSqlite() {
|
|
4490
|
+
if (cached) return cached;
|
|
4491
|
+
try {
|
|
4492
|
+
const modName = "better-sqlite3";
|
|
4493
|
+
const mod = await import(modName);
|
|
4494
|
+
cached = mod.default;
|
|
4495
|
+
return cached;
|
|
4496
|
+
} catch (err) {
|
|
4497
|
+
const detail = err instanceof Error ? err.message : String(err);
|
|
4498
|
+
throw new Error(
|
|
4499
|
+
`The local sync engine (better-sqlite3) is not installed.
|
|
4028
4500
|
|
|
4029
|
-
|
|
4501
|
+
Install it with:
|
|
4502
|
+
one sync install
|
|
4030
4503
|
|
|
4031
|
-
|
|
4504
|
+
Or manually:
|
|
4505
|
+
npm install -g better-sqlite3
|
|
4032
4506
|
|
|
4033
|
-
|
|
4034
|
-
|
|
4507
|
+
Underlying error: ${detail}`
|
|
4508
|
+
);
|
|
4509
|
+
}
|
|
4510
|
+
}
|
|
4511
|
+
async function isSqliteAvailable() {
|
|
4512
|
+
try {
|
|
4513
|
+
await loadSqlite();
|
|
4514
|
+
return true;
|
|
4515
|
+
} catch {
|
|
4516
|
+
return false;
|
|
4517
|
+
}
|
|
4518
|
+
}
|
|
4519
|
+
|
|
4520
|
+
// src/lib/sync/db.ts
|
|
4521
|
+
var DATA_DIR = path10.join(".one", "sync", "data");
|
|
4522
|
+
function listSyncedPlatforms() {
|
|
4523
|
+
if (!fs10.existsSync(DATA_DIR)) return [];
|
|
4524
|
+
return fs10.readdirSync(DATA_DIR).filter((f) => f.endsWith(".db")).map((f) => f.replace(/\.db$/, ""));
|
|
4525
|
+
}
|
|
4526
|
+
async function openDatabase(platform) {
|
|
4527
|
+
const Database = await loadSqlite();
|
|
4528
|
+
fs10.mkdirSync(DATA_DIR, { recursive: true });
|
|
4529
|
+
const dbPath = path10.join(DATA_DIR, `${platform}.db`);
|
|
4530
|
+
let db;
|
|
4531
|
+
try {
|
|
4532
|
+
db = new Database(dbPath);
|
|
4533
|
+
} catch {
|
|
4534
|
+
const backupPath = dbPath + ".bak";
|
|
4535
|
+
if (fs10.existsSync(dbPath)) {
|
|
4536
|
+
fs10.renameSync(dbPath, backupPath);
|
|
4537
|
+
process.stderr.write(`Database corrupted, starting fresh. Backup saved at ${backupPath}
|
|
4538
|
+
`);
|
|
4539
|
+
}
|
|
4540
|
+
db = new Database(dbPath);
|
|
4541
|
+
}
|
|
4542
|
+
db.pragma("journal_mode = WAL");
|
|
4543
|
+
db.pragma("busy_timeout = 15000");
|
|
4544
|
+
db.pragma("foreign_keys = OFF");
|
|
4545
|
+
return db;
|
|
4546
|
+
}
|
|
4547
|
+
function getDatabasePath(platform) {
|
|
4548
|
+
return path10.join(DATA_DIR, `${platform}.db`);
|
|
4549
|
+
}
|
|
4550
|
+
function getDatabaseSize(platform) {
|
|
4551
|
+
const dbPath = getDatabasePath(platform);
|
|
4552
|
+
if (!fs10.existsSync(dbPath)) return "0 B";
|
|
4553
|
+
const stats = fs10.statSync(dbPath);
|
|
4554
|
+
const bytes = stats.size;
|
|
4555
|
+
if (bytes < 1024) return `${bytes} B`;
|
|
4556
|
+
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
|
|
4557
|
+
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
|
4558
|
+
}
|
|
4559
|
+
function detectColumnType(value) {
|
|
4560
|
+
if (value === null || value === void 0) return "TEXT";
|
|
4561
|
+
if (typeof value === "string") return "TEXT";
|
|
4562
|
+
if (typeof value === "boolean") return "INTEGER";
|
|
4563
|
+
if (typeof value === "number") return Number.isInteger(value) ? "INTEGER" : "REAL";
|
|
4564
|
+
if (typeof value === "object") return "TEXT";
|
|
4565
|
+
return "TEXT";
|
|
4566
|
+
}
|
|
4567
|
+
function sanitizeTableName(name) {
|
|
4568
|
+
return name.replace(/[^a-zA-Z0-9_]/g, "_");
|
|
4569
|
+
}
|
|
4570
|
+
function getTableColumns(db, model) {
|
|
4571
|
+
const table = sanitizeTableName(model);
|
|
4572
|
+
const rows = db.prepare(`PRAGMA table_info("${table}")`).all();
|
|
4573
|
+
return rows.map((r) => ({ name: r.name, type: r.type }));
|
|
4574
|
+
}
|
|
4575
|
+
function tableExists(db, model) {
|
|
4576
|
+
const table = sanitizeTableName(model);
|
|
4577
|
+
const row = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name=?`).get(table);
|
|
4578
|
+
return !!row;
|
|
4579
|
+
}
|
|
4580
|
+
function ensureTable(db, model, firstRecord, idField) {
|
|
4581
|
+
const table = sanitizeTableName(model);
|
|
4582
|
+
if (tableExists(db, model)) {
|
|
4583
|
+
return getTableColumns(db, model).map((c) => c.name);
|
|
4584
|
+
}
|
|
4585
|
+
const columns = [];
|
|
4586
|
+
const colDefs = [];
|
|
4587
|
+
for (const [key, value] of Object.entries(firstRecord)) {
|
|
4588
|
+
const colType = detectColumnType(value);
|
|
4589
|
+
colDefs.push(`"${key}" ${colType}`);
|
|
4590
|
+
columns.push(key);
|
|
4591
|
+
}
|
|
4592
|
+
if (!columns.includes("_synced_at")) {
|
|
4593
|
+
colDefs.push('"_synced_at" TEXT');
|
|
4594
|
+
columns.push("_synced_at");
|
|
4595
|
+
}
|
|
4596
|
+
db.exec(`CREATE TABLE IF NOT EXISTS "${table}" (${colDefs.join(", ")})`);
|
|
4597
|
+
const safeIdField = idField.replace(/"/g, '""');
|
|
4598
|
+
db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS "idx_${table}_${sanitizeTableName(idField)}" ON "${table}" ("${safeIdField}")`);
|
|
4599
|
+
return columns;
|
|
4600
|
+
}
|
|
4601
|
+
function rebuildFtsIndex(db, model) {
|
|
4602
|
+
const table = sanitizeTableName(model);
|
|
4603
|
+
const ftsTable = `${table}_fts`;
|
|
4604
|
+
const columns = getTableColumns(db, model);
|
|
4605
|
+
const textCols = columns.filter((c) => c.type === "TEXT" && c.name !== "_synced_at").map((c) => c.name);
|
|
4606
|
+
if (textCols.length === 0) return;
|
|
4607
|
+
const quotedCols = textCols.map((c) => `"${c}"`).join(", ");
|
|
4608
|
+
db.exec(`DROP TABLE IF EXISTS "${ftsTable}"`);
|
|
4609
|
+
db.exec(`DROP TRIGGER IF EXISTS "${table}_ai"`);
|
|
4610
|
+
db.exec(`DROP TRIGGER IF EXISTS "${table}_au"`);
|
|
4611
|
+
db.exec(`CREATE VIRTUAL TABLE "${ftsTable}" USING fts5(${quotedCols})`);
|
|
4612
|
+
db.exec(`INSERT INTO "${ftsTable}"(rowid, ${quotedCols}) SELECT rowid, ${quotedCols} FROM "${table}"`);
|
|
4613
|
+
}
|
|
4614
|
+
function evolveSchema(db, model, record) {
|
|
4615
|
+
const table = sanitizeTableName(model);
|
|
4616
|
+
const existingCols = new Set(getTableColumns(db, model).map((c) => c.name));
|
|
4617
|
+
for (const [key, value] of Object.entries(record)) {
|
|
4618
|
+
if (!existingCols.has(key)) {
|
|
4619
|
+
const colType = detectColumnType(value);
|
|
4620
|
+
db.exec(`ALTER TABLE "${table}" ADD COLUMN "${key}" ${colType}`);
|
|
4621
|
+
}
|
|
4622
|
+
}
|
|
4623
|
+
}
|
|
4624
|
+
function prepareValue(value) {
|
|
4625
|
+
if (value === null || value === void 0) return null;
|
|
4626
|
+
if (typeof value === "boolean") return value ? 1 : 0;
|
|
4627
|
+
if (typeof value === "number") return value;
|
|
4628
|
+
if (typeof value === "string") return value;
|
|
4629
|
+
if (typeof value === "object") return JSON.stringify(value);
|
|
4630
|
+
return String(value);
|
|
4631
|
+
}
|
|
4632
|
+
function upsertRecords(db, model, records, idField) {
|
|
4633
|
+
if (records.length === 0) return 0;
|
|
4634
|
+
const table = sanitizeTableName(model);
|
|
4635
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
4636
|
+
const existingCols = getTableColumns(db, model).map((c) => c.name);
|
|
4637
|
+
const insertMany = db.transaction((recs) => {
|
|
4638
|
+
let count = 0;
|
|
4639
|
+
for (const record of recs) {
|
|
4640
|
+
const recordKeys = Object.keys(record);
|
|
4641
|
+
const newKeys = recordKeys.filter((k) => !existingCols.includes(k));
|
|
4642
|
+
if (newKeys.length > 0) {
|
|
4643
|
+
for (const key of newKeys) {
|
|
4644
|
+
const colType = detectColumnType(record[key]);
|
|
4645
|
+
db.exec(`ALTER TABLE "${table}" ADD COLUMN "${key}" ${colType}`);
|
|
4646
|
+
existingCols.push(key);
|
|
4647
|
+
}
|
|
4648
|
+
}
|
|
4649
|
+
const fullRecord = { ...record, _synced_at: now };
|
|
4650
|
+
const cols = Object.keys(fullRecord).filter((k) => existingCols.includes(k) || k === "_synced_at");
|
|
4651
|
+
const quotedCols = cols.map((c) => `"${c}"`).join(", ");
|
|
4652
|
+
const placeholders = cols.map(() => "?").join(", ");
|
|
4653
|
+
const values = cols.map((c) => prepareValue(fullRecord[c]));
|
|
4654
|
+
const safeIdField = idField.replace(/"/g, '""');
|
|
4655
|
+
const updateCols = cols.filter((c) => c !== idField).map((c) => `"${c}" = excluded."${c}"`).join(", ");
|
|
4656
|
+
db.prepare(
|
|
4657
|
+
`INSERT INTO "${table}" (${quotedCols}) VALUES (${placeholders}) ON CONFLICT("${safeIdField}") DO UPDATE SET ${updateCols}`
|
|
4658
|
+
).run(...values);
|
|
4659
|
+
count++;
|
|
4660
|
+
}
|
|
4661
|
+
return count;
|
|
4662
|
+
});
|
|
4663
|
+
return insertMany(records);
|
|
4664
|
+
}
|
|
4665
|
+
function deleteRecords(db, model, where, params) {
|
|
4666
|
+
const table = sanitizeTableName(model);
|
|
4667
|
+
const result = db.prepare(`DELETE FROM "${table}" WHERE ${where}`).run(...params);
|
|
4668
|
+
return result.changes;
|
|
4669
|
+
}
|
|
4670
|
+
function dropTable(db, model) {
|
|
4671
|
+
const table = sanitizeTableName(model);
|
|
4672
|
+
db.exec(`DROP TABLE IF EXISTS "${table}_fts"`);
|
|
4673
|
+
db.exec(`DROP TRIGGER IF EXISTS "${table}_ai"`);
|
|
4674
|
+
db.exec(`DROP TRIGGER IF EXISTS "${table}_au"`);
|
|
4675
|
+
db.exec(`DROP TABLE IF EXISTS "${table}"`);
|
|
4676
|
+
}
|
|
4677
|
+
function listTables(db) {
|
|
4678
|
+
const rows = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE '%_fts%' AND name NOT LIKE 'sqlite_%'`).all();
|
|
4679
|
+
return rows.map((r) => r.name);
|
|
4680
|
+
}
|
|
4681
|
+
function countRecords(db, model) {
|
|
4682
|
+
const table = sanitizeTableName(model);
|
|
4683
|
+
if (!tableExists(db, model)) return 0;
|
|
4684
|
+
const row = db.prepare(`SELECT COUNT(*) as count FROM "${table}"`).get();
|
|
4685
|
+
return row.count;
|
|
4686
|
+
}
|
|
4687
|
+
function deleteDatabase(platform) {
|
|
4688
|
+
const dbPath = getDatabasePath(platform);
|
|
4689
|
+
if (fs10.existsSync(dbPath)) fs10.unlinkSync(dbPath);
|
|
4690
|
+
if (fs10.existsSync(dbPath + "-wal")) fs10.unlinkSync(dbPath + "-wal");
|
|
4691
|
+
if (fs10.existsSync(dbPath + "-shm")) fs10.unlinkSync(dbPath + "-shm");
|
|
4692
|
+
}
|
|
4693
|
+
|
|
4694
|
+
// src/lib/sync/lock.ts
|
|
4695
|
+
import fs11 from "fs";
|
|
4696
|
+
import path11 from "path";
|
|
4697
|
+
var LOCK_DIR_REL = path11.join(".one", "sync", "locks");
|
|
4698
|
+
var STALE_MS = 30 * 60 * 1e3;
|
|
4699
|
+
function lockPath(platform, model) {
|
|
4700
|
+
return path11.join(LOCK_DIR_REL, `${platform}_${model}`);
|
|
4701
|
+
}
|
|
4702
|
+
function isProcessAlive(pid) {
|
|
4703
|
+
try {
|
|
4704
|
+
process.kill(pid, 0);
|
|
4705
|
+
return true;
|
|
4706
|
+
} catch {
|
|
4707
|
+
return false;
|
|
4708
|
+
}
|
|
4709
|
+
}
|
|
4710
|
+
var SyncLockError = class extends Error {
|
|
4711
|
+
constructor(message) {
|
|
4712
|
+
super(message);
|
|
4713
|
+
this.name = "SyncLockError";
|
|
4714
|
+
}
|
|
4715
|
+
};
|
|
4716
|
+
function acquireSyncLock(platform, model) {
|
|
4717
|
+
fs11.mkdirSync(LOCK_DIR_REL, { recursive: true });
|
|
4718
|
+
const dir = lockPath(platform, model);
|
|
4719
|
+
const pidFile = path11.join(dir, "pid");
|
|
4720
|
+
try {
|
|
4721
|
+
fs11.mkdirSync(dir);
|
|
4722
|
+
} catch (err) {
|
|
4723
|
+
const stat = (() => {
|
|
4724
|
+
try {
|
|
4725
|
+
return fs11.statSync(dir);
|
|
4726
|
+
} catch {
|
|
4727
|
+
return null;
|
|
4728
|
+
}
|
|
4729
|
+
})();
|
|
4730
|
+
if (stat) {
|
|
4731
|
+
const age = Date.now() - stat.mtimeMs;
|
|
4732
|
+
let ownerPid = null;
|
|
4733
|
+
try {
|
|
4734
|
+
const raw = fs11.readFileSync(pidFile, "utf-8");
|
|
4735
|
+
const parsed = parseInt(raw.trim(), 10);
|
|
4736
|
+
if (!isNaN(parsed)) ownerPid = parsed;
|
|
4737
|
+
} catch {
|
|
4738
|
+
}
|
|
4739
|
+
const ownerDead = ownerPid !== null && !isProcessAlive(ownerPid);
|
|
4740
|
+
const veryOld = age > STALE_MS;
|
|
4741
|
+
if (ownerDead || veryOld) {
|
|
4742
|
+
try {
|
|
4743
|
+
fs11.rmSync(dir, { recursive: true, force: true });
|
|
4744
|
+
fs11.mkdirSync(dir);
|
|
4745
|
+
} catch {
|
|
4746
|
+
throw new SyncLockError(
|
|
4747
|
+
`Could not take over stale lock at ${dir}. Remove it manually if no sync is running.`
|
|
4748
|
+
);
|
|
4749
|
+
}
|
|
4750
|
+
} else {
|
|
4751
|
+
const ownerMsg = ownerPid !== null ? ` (held by pid ${ownerPid})` : "";
|
|
4752
|
+
throw new SyncLockError(
|
|
4753
|
+
`Another sync for ${platform}/${model} is already running${ownerMsg}. Wait for it to finish, or remove ${dir} manually if you're sure it's stale.`
|
|
4754
|
+
);
|
|
4755
|
+
}
|
|
4756
|
+
} else {
|
|
4757
|
+
throw new SyncLockError(`Failed to acquire sync lock: ${err instanceof Error ? err.message : String(err)}`);
|
|
4758
|
+
}
|
|
4759
|
+
}
|
|
4760
|
+
try {
|
|
4761
|
+
fs11.writeFileSync(pidFile, String(process.pid));
|
|
4762
|
+
} catch {
|
|
4763
|
+
}
|
|
4764
|
+
return {
|
|
4765
|
+
release() {
|
|
4766
|
+
try {
|
|
4767
|
+
fs11.rmSync(dir, { recursive: true, force: true });
|
|
4768
|
+
} catch {
|
|
4769
|
+
}
|
|
4770
|
+
}
|
|
4771
|
+
};
|
|
4772
|
+
}
|
|
4773
|
+
|
|
4774
|
+
// src/lib/sync/hooks.ts
|
|
4775
|
+
import { spawn as spawn2 } from "child_process";
|
|
4776
|
+
import fs12 from "fs";
|
|
4777
|
+
import path12 from "path";
|
|
4778
|
+
var EVENTS_DIR = path12.join(".one", "sync", "events");
|
|
4779
|
+
function classifyRecords(db, model, records, idField, tableExists2) {
|
|
4780
|
+
if (!tableExists2 || records.length === 0) {
|
|
4781
|
+
return { inserts: records, updates: [] };
|
|
4782
|
+
}
|
|
4783
|
+
const safeTable = model.replace(/[^a-zA-Z0-9_]/g, "_");
|
|
4784
|
+
const safeIdField = idField.replace(/"/g, '""');
|
|
4785
|
+
const ids = records.map((r) => r[idField]).filter((id) => id !== void 0 && id !== null);
|
|
4786
|
+
if (ids.length === 0) return { inserts: records, updates: [] };
|
|
4787
|
+
const CHUNK = 500;
|
|
4788
|
+
const existingIds = /* @__PURE__ */ new Set();
|
|
4789
|
+
for (let i = 0; i < ids.length; i += CHUNK) {
|
|
4790
|
+
const chunk = ids.slice(i, i + CHUNK);
|
|
4791
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
4792
|
+
const rows = db.prepare(
|
|
4793
|
+
`SELECT "${safeIdField}" as id FROM "${safeTable}" WHERE "${safeIdField}" IN (${placeholders})`
|
|
4794
|
+
).all(...chunk);
|
|
4795
|
+
for (const row of rows) existingIds.add(row.id);
|
|
4796
|
+
}
|
|
4797
|
+
const inserts = [];
|
|
4798
|
+
const updates = [];
|
|
4799
|
+
for (const record of records) {
|
|
4800
|
+
const id = record[idField];
|
|
4801
|
+
if (typeof id === "string" || typeof id === "number") {
|
|
4802
|
+
if (existingIds.has(id)) {
|
|
4803
|
+
updates.push(record);
|
|
4804
|
+
} else {
|
|
4805
|
+
inserts.push(record);
|
|
4806
|
+
}
|
|
4807
|
+
} else {
|
|
4808
|
+
inserts.push(record);
|
|
4809
|
+
}
|
|
4810
|
+
}
|
|
4811
|
+
return { inserts, updates };
|
|
4812
|
+
}
|
|
4813
|
+
async function fireHooks(hookCommand, events) {
|
|
4814
|
+
if (events.length === 0) return;
|
|
4815
|
+
if (hookCommand === "log") {
|
|
4816
|
+
appendEventLog(events);
|
|
4817
|
+
return;
|
|
4818
|
+
}
|
|
4819
|
+
await runShellHook(hookCommand, events);
|
|
4820
|
+
}
|
|
4821
|
+
function appendEventLog(events) {
|
|
4822
|
+
if (events.length === 0) return;
|
|
4823
|
+
const { platform, model } = events[0];
|
|
4824
|
+
fs12.mkdirSync(EVENTS_DIR, { recursive: true });
|
|
4825
|
+
const logPath = path12.join(EVENTS_DIR, `${platform}_${model}.jsonl`);
|
|
4826
|
+
const lines = events.map((e) => JSON.stringify(e)).join("\n") + "\n";
|
|
4827
|
+
fs12.appendFileSync(logPath, lines);
|
|
4828
|
+
}
|
|
4829
|
+
function runShellHook(command, events) {
|
|
4830
|
+
return new Promise((resolve) => {
|
|
4831
|
+
const input = events.map((e) => JSON.stringify(e)).join("\n") + "\n";
|
|
4832
|
+
const child = spawn2("sh", ["-c", command], {
|
|
4833
|
+
stdio: ["pipe", "ignore", "ignore"]
|
|
4834
|
+
});
|
|
4835
|
+
child.stdin.write(input);
|
|
4836
|
+
child.stdin.end();
|
|
4837
|
+
child.on("exit", () => resolve());
|
|
4838
|
+
child.on("error", () => resolve());
|
|
4839
|
+
setTimeout(() => {
|
|
4840
|
+
try {
|
|
4841
|
+
child.kill();
|
|
4842
|
+
} catch {
|
|
4843
|
+
}
|
|
4844
|
+
resolve();
|
|
4845
|
+
}, 3e4);
|
|
4846
|
+
});
|
|
4847
|
+
}
|
|
4848
|
+
|
|
4849
|
+
// src/lib/sync/enrich.ts
|
|
4850
|
+
var DEFAULT_CONCURRENCY = 5;
|
|
4851
|
+
var MAX_RETRIES = 3;
|
|
4852
|
+
var BASE_BACKOFF_MS = 2e3;
|
|
4853
|
+
function sleep(ms) {
|
|
4854
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
4855
|
+
}
|
|
4856
|
+
function interpolate(template, record) {
|
|
4857
|
+
return template.replace(/\{?\{(\w+(?:\.\w+)*)\}\}?/g, (_, path16) => {
|
|
4858
|
+
const parts = path16.split(".");
|
|
4859
|
+
let value = record;
|
|
4860
|
+
for (const part of parts) {
|
|
4861
|
+
if (typeof value !== "object" || value === null) return "";
|
|
4862
|
+
value = value[part];
|
|
4863
|
+
}
|
|
4864
|
+
return value === null || value === void 0 ? "" : String(value);
|
|
4865
|
+
});
|
|
4866
|
+
}
|
|
4867
|
+
function interpolateParams(template, record) {
|
|
4868
|
+
if (!template) return void 0;
|
|
4869
|
+
const out = {};
|
|
4870
|
+
for (const [key, value] of Object.entries(template)) {
|
|
4871
|
+
out[key] = typeof value === "string" ? interpolate(value, record) : value;
|
|
4872
|
+
}
|
|
4873
|
+
return out;
|
|
4874
|
+
}
|
|
4875
|
+
function deepMerge(target, source) {
|
|
4876
|
+
const result = { ...target };
|
|
4877
|
+
for (const [key, value] of Object.entries(source)) {
|
|
4878
|
+
if (value !== null && typeof value === "object" && !Array.isArray(value) && typeof result[key] === "object" && result[key] !== null && !Array.isArray(result[key])) {
|
|
4879
|
+
result[key] = deepMerge(result[key], value);
|
|
4880
|
+
} else {
|
|
4881
|
+
result[key] = value;
|
|
4882
|
+
}
|
|
4883
|
+
}
|
|
4884
|
+
return result;
|
|
4885
|
+
}
|
|
4886
|
+
function getByDotPath2(obj, path16) {
|
|
4887
|
+
const parts = path16.split(".");
|
|
4888
|
+
let current = obj;
|
|
4889
|
+
for (const part of parts) {
|
|
4890
|
+
if (current === null || current === void 0 || typeof current !== "object") return void 0;
|
|
4891
|
+
current = current[part];
|
|
4892
|
+
}
|
|
4893
|
+
return current;
|
|
4894
|
+
}
|
|
4895
|
+
function stripExcludedFields(obj, paths) {
|
|
4896
|
+
for (const path16 of paths) {
|
|
4897
|
+
stripOnePath(obj, path16.replace(/\[\]/g, ".*").split("."));
|
|
4898
|
+
}
|
|
4899
|
+
}
|
|
4900
|
+
function stripOnePath(obj, parts) {
|
|
4901
|
+
if (parts.length === 0 || !obj || typeof obj !== "object") return;
|
|
4902
|
+
const [current, ...rest] = parts;
|
|
4903
|
+
if (current === "*") {
|
|
4904
|
+
for (const value of Object.values(obj)) {
|
|
4905
|
+
if (Array.isArray(value)) {
|
|
4906
|
+
for (const item of value) {
|
|
4907
|
+
if (typeof item === "object" && item !== null) {
|
|
4908
|
+
stripOnePath(item, rest);
|
|
4909
|
+
}
|
|
4910
|
+
}
|
|
4911
|
+
}
|
|
4912
|
+
}
|
|
4913
|
+
return;
|
|
4914
|
+
}
|
|
4915
|
+
if (rest.length === 0) {
|
|
4916
|
+
delete obj[current];
|
|
4917
|
+
return;
|
|
4918
|
+
}
|
|
4919
|
+
const child = obj[current];
|
|
4920
|
+
if (Array.isArray(child) && rest[0] === "*") {
|
|
4921
|
+
for (const item of child) {
|
|
4922
|
+
if (typeof item === "object" && item !== null) {
|
|
4923
|
+
stripOnePath(item, rest.slice(1));
|
|
4924
|
+
}
|
|
4925
|
+
}
|
|
4926
|
+
} else if (typeof child === "object" && child !== null && !Array.isArray(child)) {
|
|
4927
|
+
stripOnePath(child, rest);
|
|
4928
|
+
}
|
|
4929
|
+
}
|
|
4930
|
+
function pickFields(obj, fields) {
|
|
4931
|
+
const result = {};
|
|
4932
|
+
for (const field of fields) {
|
|
4933
|
+
const value = getByDotPath2(obj, field);
|
|
4934
|
+
if (value !== void 0) result[field] = value;
|
|
4935
|
+
}
|
|
4936
|
+
return result;
|
|
4937
|
+
}
|
|
4938
|
+
async function enrichPhase(api, db, config2, model, idField, connectionKey, platform) {
|
|
4939
|
+
const startTime = Date.now();
|
|
4940
|
+
const tsField = config2.timestampField ?? "_enriched_at";
|
|
4941
|
+
const safeTable = model.replace(/[^a-zA-Z0-9_]/g, "_");
|
|
4942
|
+
const safeIdField = idField.replace(/"/g, '""');
|
|
4943
|
+
const cols = db.prepare(`PRAGMA table_info("${safeTable}")`).all();
|
|
4944
|
+
if (!cols.some((c) => c.name === tsField)) {
|
|
4945
|
+
db.exec(`ALTER TABLE "${safeTable}" ADD COLUMN "${tsField}" TEXT`);
|
|
4946
|
+
}
|
|
4947
|
+
const unenriched = db.prepare(
|
|
4948
|
+
`SELECT * FROM "${safeTable}" WHERE "${tsField}" IS NULL`
|
|
4949
|
+
).all();
|
|
4950
|
+
const total = unenriched.length;
|
|
4951
|
+
if (total === 0) {
|
|
4952
|
+
return { enriched: 0, skipped: 0, rateLimited: 0, total: 0, duration: "0s" };
|
|
4953
|
+
}
|
|
4954
|
+
for (const row of unenriched) {
|
|
4955
|
+
for (const [key, value] of Object.entries(row)) {
|
|
4956
|
+
if (typeof value === "string" && (value.startsWith("{") || value.startsWith("["))) {
|
|
4957
|
+
try {
|
|
4958
|
+
row[key] = JSON.parse(value);
|
|
4959
|
+
} catch {
|
|
4960
|
+
}
|
|
4961
|
+
}
|
|
4962
|
+
}
|
|
4963
|
+
}
|
|
4964
|
+
let detailAction;
|
|
4965
|
+
try {
|
|
4966
|
+
detailAction = await api.getActionDetails(config2.actionId);
|
|
4967
|
+
} catch (err) {
|
|
4968
|
+
throw new Error(
|
|
4969
|
+
`Enrich: could not load action ${config2.actionId}: ${err instanceof Error ? err.message : String(err)}`
|
|
4970
|
+
);
|
|
4971
|
+
}
|
|
4972
|
+
let concurrency = config2.concurrency ?? DEFAULT_CONCURRENCY;
|
|
4973
|
+
let enriched = 0;
|
|
4974
|
+
let skipped = 0;
|
|
4975
|
+
let rateLimited = 0;
|
|
4976
|
+
for (let i = 0; i < unenriched.length; i += concurrency) {
|
|
4977
|
+
const batch = unenriched.slice(i, i + concurrency);
|
|
4978
|
+
if (!isAgentMode()) {
|
|
4979
|
+
process.stderr.write(` Enriching ${platform}/${model}... ${i}/${total}\r`);
|
|
4980
|
+
}
|
|
4981
|
+
const results = await Promise.allSettled(
|
|
4982
|
+
batch.map((row) => enrichSingleRow(api, detailAction, config2, row, connectionKey, platform))
|
|
4983
|
+
);
|
|
4984
|
+
let batchHitRateLimit = false;
|
|
4985
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
4986
|
+
for (let j = 0; j < results.length; j++) {
|
|
4987
|
+
const result = results[j];
|
|
4988
|
+
const row = batch[j];
|
|
4989
|
+
const id = row[idField];
|
|
4990
|
+
if (result.status === "fulfilled" && result.value !== null) {
|
|
4991
|
+
let enrichedData = result.value;
|
|
4992
|
+
if (config2.fields && config2.fields.length > 0) {
|
|
4993
|
+
enrichedData = pickFields(enrichedData, config2.fields);
|
|
4994
|
+
}
|
|
4995
|
+
if (config2.exclude && config2.exclude.length > 0) {
|
|
4996
|
+
stripExcludedFields(enrichedData, config2.exclude);
|
|
4997
|
+
}
|
|
4998
|
+
const merged = config2.merge !== false ? deepMerge(row, enrichedData) : { ...enrichedData, [idField]: id };
|
|
4999
|
+
merged[tsField] = now;
|
|
5000
|
+
const setClauses = [];
|
|
5001
|
+
const values = [];
|
|
5002
|
+
for (const [key, val] of Object.entries(merged)) {
|
|
5003
|
+
if (key === idField) continue;
|
|
5004
|
+
setClauses.push(`"${key}" = ?`);
|
|
5005
|
+
values.push(prepareValue2(val));
|
|
5006
|
+
}
|
|
5007
|
+
values.push(prepareValue2(id));
|
|
5008
|
+
if (setClauses.length > 0) {
|
|
5009
|
+
const existingCols = new Set(db.prepare(`PRAGMA table_info("${safeTable}")`).all().map((c) => c.name));
|
|
5010
|
+
for (const [key, val] of Object.entries(merged)) {
|
|
5011
|
+
if (!existingCols.has(key)) {
|
|
5012
|
+
const colType = detectColumnType2(val);
|
|
5013
|
+
db.exec(`ALTER TABLE "${safeTable}" ADD COLUMN "${key}" ${colType}`);
|
|
5014
|
+
existingCols.add(key);
|
|
5015
|
+
}
|
|
5016
|
+
}
|
|
5017
|
+
db.prepare(
|
|
5018
|
+
`UPDATE "${safeTable}" SET ${setClauses.join(", ")} WHERE "${safeIdField}" = ?`
|
|
5019
|
+
).run(...values);
|
|
5020
|
+
}
|
|
5021
|
+
enriched++;
|
|
5022
|
+
} else if (result.status === "fulfilled" && result.value === null) {
|
|
5023
|
+
rateLimited++;
|
|
5024
|
+
skipped++;
|
|
5025
|
+
batchHitRateLimit = true;
|
|
5026
|
+
} else {
|
|
5027
|
+
skipped++;
|
|
5028
|
+
}
|
|
5029
|
+
}
|
|
5030
|
+
if (batchHitRateLimit) {
|
|
5031
|
+
concurrency = Math.max(1, Math.floor(concurrency / 2));
|
|
5032
|
+
if (!isAgentMode()) {
|
|
5033
|
+
process.stderr.write(` Enrich: rate limited \u2014 reducing concurrency to ${concurrency}
|
|
5034
|
+
`);
|
|
5035
|
+
}
|
|
5036
|
+
await sleep(BASE_BACKOFF_MS * 4);
|
|
5037
|
+
} else if (i + concurrency < unenriched.length) {
|
|
5038
|
+
await sleep(config2.delayMs ?? 200);
|
|
5039
|
+
}
|
|
5040
|
+
}
|
|
5041
|
+
if (!isAgentMode()) {
|
|
5042
|
+
process.stderr.write(` Enriching ${platform}/${model}... ${enriched}/${total} done
|
|
5043
|
+
`);
|
|
5044
|
+
}
|
|
5045
|
+
const elapsed = Date.now() - startTime;
|
|
5046
|
+
const duration = elapsed < 1e3 ? `${elapsed}ms` : elapsed < 6e4 ? `${(elapsed / 1e3).toFixed(1)}s` : `${Math.floor(elapsed / 6e4)}m ${Math.floor(elapsed % 6e4 / 1e3)}s`;
|
|
5047
|
+
return { enriched, skipped, rateLimited, total, duration };
|
|
5048
|
+
}
|
|
5049
|
+
async function enrichSingleRow(api, detailAction, config2, row, connectionKey, platform) {
|
|
5050
|
+
const pathVars = interpolateParams(config2.pathVars, row);
|
|
5051
|
+
const queryParams = interpolateParams(config2.queryParams, row);
|
|
5052
|
+
const body = config2.body ? JSON.parse(interpolate(JSON.stringify(config2.body), row)) : void 0;
|
|
5053
|
+
for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
|
|
5054
|
+
try {
|
|
5055
|
+
const result = await api.executePassthroughRequest({
|
|
5056
|
+
platform,
|
|
5057
|
+
actionId: config2.actionId,
|
|
5058
|
+
connectionKey,
|
|
5059
|
+
pathVariables: pathVars,
|
|
5060
|
+
queryParams,
|
|
5061
|
+
data: body
|
|
5062
|
+
}, detailAction);
|
|
5063
|
+
let detailData;
|
|
5064
|
+
if (config2.resultsPath) {
|
|
5065
|
+
const extracted = getByDotPath2(result.responseData, config2.resultsPath);
|
|
5066
|
+
detailData = typeof extracted === "object" && extracted !== null && !Array.isArray(extracted) ? extracted : { _enriched: extracted };
|
|
5067
|
+
} else {
|
|
5068
|
+
detailData = typeof result.responseData === "object" && result.responseData !== null ? result.responseData : {};
|
|
5069
|
+
}
|
|
5070
|
+
return detailData;
|
|
5071
|
+
} catch (err) {
|
|
5072
|
+
if (err instanceof ApiError && err.status === 429) {
|
|
5073
|
+
const retryAfter = err.retryAfterSeconds ?? Math.min(BASE_BACKOFF_MS / 1e3 * Math.pow(2, attempt), 60);
|
|
5074
|
+
await sleep(retryAfter * 1e3);
|
|
5075
|
+
if (attempt === MAX_RETRIES - 1) return null;
|
|
5076
|
+
continue;
|
|
5077
|
+
}
|
|
5078
|
+
if (err instanceof ApiError && (err.status >= 500 && err.status <= 504)) {
|
|
5079
|
+
await sleep(Math.min(3 * Math.pow(2, attempt), 20) * 1e3);
|
|
5080
|
+
if (attempt === MAX_RETRIES - 1) return null;
|
|
5081
|
+
continue;
|
|
5082
|
+
}
|
|
5083
|
+
if (err instanceof ApiError && (err.status === 401 || err.status === 403)) {
|
|
5084
|
+
throw err;
|
|
5085
|
+
}
|
|
5086
|
+
if (attempt === MAX_RETRIES - 1) return null;
|
|
5087
|
+
await sleep(BASE_BACKOFF_MS * Math.pow(2, attempt));
|
|
5088
|
+
}
|
|
5089
|
+
}
|
|
5090
|
+
return null;
|
|
5091
|
+
}
|
|
5092
|
+
function prepareValue2(value) {
|
|
5093
|
+
if (value === null || value === void 0) return null;
|
|
5094
|
+
if (typeof value === "boolean") return value ? 1 : 0;
|
|
5095
|
+
if (typeof value === "number") return value;
|
|
5096
|
+
if (typeof value === "string") return value;
|
|
5097
|
+
if (typeof value === "object") return JSON.stringify(value);
|
|
5098
|
+
return String(value);
|
|
5099
|
+
}
|
|
5100
|
+
function detectColumnType2(value) {
|
|
5101
|
+
if (value === null || value === void 0) return "TEXT";
|
|
5102
|
+
if (typeof value === "string") return "TEXT";
|
|
5103
|
+
if (typeof value === "boolean") return "INTEGER";
|
|
5104
|
+
if (typeof value === "number") return Number.isInteger(value) ? "INTEGER" : "REAL";
|
|
5105
|
+
if (typeof value === "object") return "TEXT";
|
|
5106
|
+
return "TEXT";
|
|
5107
|
+
}
|
|
5108
|
+
|
|
5109
|
+
// src/lib/sync/transform.ts
|
|
5110
|
+
import { spawn as spawn3 } from "child_process";
|
|
5111
|
+
var TRANSFORM_TIMEOUT_MS = 6e4;
|
|
5112
|
+
async function transformRecords(command, records) {
|
|
5113
|
+
const input = JSON.stringify(records);
|
|
5114
|
+
return new Promise((resolve) => {
|
|
5115
|
+
const child = spawn3("sh", ["-c", command], {
|
|
5116
|
+
stdio: ["pipe", "pipe", "pipe"]
|
|
5117
|
+
});
|
|
5118
|
+
let stdout = "";
|
|
5119
|
+
let stderr = "";
|
|
5120
|
+
child.stdout.on("data", (chunk) => {
|
|
5121
|
+
stdout += chunk.toString();
|
|
5122
|
+
});
|
|
5123
|
+
child.stderr.on("data", (chunk) => {
|
|
5124
|
+
stderr += chunk.toString();
|
|
5125
|
+
});
|
|
5126
|
+
const timer = setTimeout(() => {
|
|
5127
|
+
try {
|
|
5128
|
+
child.kill();
|
|
5129
|
+
} catch {
|
|
5130
|
+
}
|
|
5131
|
+
if (!isAgentMode()) {
|
|
5132
|
+
process.stderr.write(` Transform timed out after ${TRANSFORM_TIMEOUT_MS / 1e3}s \u2014 using original records
|
|
5133
|
+
`);
|
|
5134
|
+
}
|
|
5135
|
+
resolve(null);
|
|
5136
|
+
}, TRANSFORM_TIMEOUT_MS);
|
|
5137
|
+
child.on("exit", (code) => {
|
|
5138
|
+
clearTimeout(timer);
|
|
5139
|
+
if (stderr.trim() && !isAgentMode()) {
|
|
5140
|
+
process.stderr.write(` Transform stderr: ${stderr.trim()}
|
|
5141
|
+
`);
|
|
5142
|
+
}
|
|
5143
|
+
if (code !== 0) {
|
|
5144
|
+
if (!isAgentMode()) {
|
|
5145
|
+
process.stderr.write(` Transform exited with code ${code} \u2014 using original records
|
|
5146
|
+
`);
|
|
5147
|
+
}
|
|
5148
|
+
resolve(null);
|
|
5149
|
+
return;
|
|
5150
|
+
}
|
|
5151
|
+
try {
|
|
5152
|
+
const parsed = JSON.parse(stdout.trim());
|
|
5153
|
+
if (!Array.isArray(parsed)) {
|
|
5154
|
+
if (!isAgentMode()) {
|
|
5155
|
+
process.stderr.write(` Transform returned non-array JSON \u2014 using original records
|
|
5156
|
+
`);
|
|
5157
|
+
}
|
|
5158
|
+
resolve(null);
|
|
5159
|
+
return;
|
|
5160
|
+
}
|
|
5161
|
+
resolve(parsed);
|
|
5162
|
+
} catch {
|
|
5163
|
+
if (!isAgentMode()) {
|
|
5164
|
+
process.stderr.write(` Transform returned invalid JSON \u2014 using original records
|
|
5165
|
+
`);
|
|
5166
|
+
}
|
|
5167
|
+
resolve(null);
|
|
5168
|
+
}
|
|
5169
|
+
});
|
|
5170
|
+
child.on("error", (err) => {
|
|
5171
|
+
clearTimeout(timer);
|
|
5172
|
+
if (!isAgentMode()) {
|
|
5173
|
+
process.stderr.write(` Transform failed to start: ${err.message} \u2014 using original records
|
|
5174
|
+
`);
|
|
5175
|
+
}
|
|
5176
|
+
resolve(null);
|
|
5177
|
+
});
|
|
5178
|
+
child.stdin.write(input);
|
|
5179
|
+
child.stdin.end();
|
|
5180
|
+
});
|
|
5181
|
+
}
|
|
5182
|
+
|
|
5183
|
+
// src/lib/sync/runner.ts
|
|
5184
|
+
var MAX_RETRIES_PER_PAGE = 3;
|
|
5185
|
+
var DEFAULT_SINCE_DAYS = 90;
|
|
5186
|
+
function isNetworkError(err) {
|
|
5187
|
+
if (!(err instanceof Error)) return false;
|
|
5188
|
+
const code = err.code;
|
|
5189
|
+
if (code && ["ECONNRESET", "ETIMEDOUT", "ENOTFOUND", "ECONNREFUSED", "EPIPE", "UND_ERR_SOCKET"].includes(code)) {
|
|
5190
|
+
return true;
|
|
5191
|
+
}
|
|
5192
|
+
const msg = err.message.toLowerCase();
|
|
5193
|
+
return msg.includes("fetch failed") || msg.includes("network") || msg.includes("socket");
|
|
5194
|
+
}
|
|
5195
|
+
function truncate(str, maxLen) {
|
|
5196
|
+
if (str.length <= maxLen) return str;
|
|
5197
|
+
return str.slice(0, maxLen) + "\u2026(truncated)";
|
|
5198
|
+
}
|
|
5199
|
+
function sleep2(ms) {
|
|
5200
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
5201
|
+
}
|
|
5202
|
+
function stripFields(record, paths) {
|
|
5203
|
+
for (const path16 of paths) {
|
|
5204
|
+
stripOnePath2(record, path16.split("."));
|
|
5205
|
+
}
|
|
5206
|
+
}
|
|
5207
|
+
function stripOnePath2(obj, parts) {
|
|
5208
|
+
if (parts.length === 0 || !obj || typeof obj !== "object") return;
|
|
5209
|
+
const current = parts[0];
|
|
5210
|
+
const rest = parts.slice(1);
|
|
5211
|
+
if (current.endsWith("[]")) {
|
|
5212
|
+
const key = current.slice(0, -2);
|
|
5213
|
+
const arr = obj[key];
|
|
5214
|
+
if (Array.isArray(arr)) {
|
|
5215
|
+
if (rest.length === 0) {
|
|
5216
|
+
delete obj[key];
|
|
5217
|
+
} else {
|
|
5218
|
+
for (const item of arr) {
|
|
5219
|
+
if (typeof item === "object" && item !== null) {
|
|
5220
|
+
stripOnePath2(item, rest);
|
|
5221
|
+
}
|
|
5222
|
+
}
|
|
5223
|
+
}
|
|
5224
|
+
}
|
|
5225
|
+
return;
|
|
5226
|
+
}
|
|
5227
|
+
if (rest.length === 0) {
|
|
5228
|
+
delete obj[current];
|
|
5229
|
+
return;
|
|
5230
|
+
}
|
|
5231
|
+
const child = obj[current];
|
|
5232
|
+
if (typeof child === "object" && child !== null && !Array.isArray(child)) {
|
|
5233
|
+
stripOnePath2(child, rest);
|
|
5234
|
+
}
|
|
5235
|
+
}
|
|
5236
|
+
function parseSince(since) {
|
|
5237
|
+
const durationMatch = since.match(/^(\d+)([dhm])$/);
|
|
5238
|
+
if (durationMatch) {
|
|
5239
|
+
const amount = parseInt(durationMatch[1], 10);
|
|
5240
|
+
const unit = durationMatch[2];
|
|
5241
|
+
const now = /* @__PURE__ */ new Date();
|
|
5242
|
+
if (unit === "d") now.setDate(now.getDate() - amount);
|
|
5243
|
+
else if (unit === "h") now.setHours(now.getHours() - amount);
|
|
5244
|
+
else if (unit === "m") now.setMonth(now.getMonth() - amount);
|
|
5245
|
+
return now;
|
|
5246
|
+
}
|
|
5247
|
+
return new Date(since);
|
|
5248
|
+
}
|
|
5249
|
+
function formatDuration(ms) {
|
|
5250
|
+
if (ms < 1e3) return `${ms}ms`;
|
|
5251
|
+
const seconds = ms / 1e3;
|
|
5252
|
+
if (seconds < 60) return `${seconds.toFixed(1)}s`;
|
|
5253
|
+
const minutes = Math.floor(seconds / 60);
|
|
5254
|
+
const remainingSeconds = (seconds % 60).toFixed(0);
|
|
5255
|
+
return `${minutes}m ${remainingSeconds}s`;
|
|
5256
|
+
}
|
|
5257
|
+
async function syncModel(api, profile, options) {
|
|
5258
|
+
const { platform, model } = profile;
|
|
5259
|
+
const startTime = Date.now();
|
|
5260
|
+
if (options.fullRefresh && options.since) {
|
|
5261
|
+
throw new Error(
|
|
5262
|
+
"--full-refresh and --since cannot be used together. --full-refresh always fetches the whole collection."
|
|
5263
|
+
);
|
|
5264
|
+
}
|
|
5265
|
+
const lock = options.dryRun ? null : acquireSyncLock(platform, model);
|
|
5266
|
+
const existingState = getModelState(platform, model);
|
|
5267
|
+
if (existingState?.status === "syncing" && !options.force) {
|
|
5268
|
+
if (lock) lock.release();
|
|
5269
|
+
throw new Error(
|
|
5270
|
+
`Sync state says ${platform}/${model} is already syncing. Use --force to override (this may happen if a previous sync crashed before cleanup).`
|
|
5271
|
+
);
|
|
5272
|
+
}
|
|
5273
|
+
if (!profile.dateFilter && !options.dryRun && !options.fullRefresh && !isAgentMode()) {
|
|
5274
|
+
process.stderr.write(
|
|
5275
|
+
`\u26A0 ${platform}/${model} profile has no dateFilter \u2014 this sync will fetch the entire collection every run. Add a dateFilter to the profile for true incremental sync, or accept the full-pull cost.
|
|
5276
|
+
`
|
|
5277
|
+
);
|
|
5278
|
+
}
|
|
5279
|
+
if (!options.dryRun) {
|
|
5280
|
+
updateModelState(platform, model, { status: "syncing" });
|
|
5281
|
+
}
|
|
5282
|
+
let db = null;
|
|
5283
|
+
let totalRecords = 0;
|
|
5284
|
+
let pagesProcessed = 0;
|
|
5285
|
+
let lastCursor = null;
|
|
5286
|
+
const seenIds = /* @__PURE__ */ new Set();
|
|
5287
|
+
let hooksInserted = 0;
|
|
5288
|
+
let hooksUpdated = 0;
|
|
5289
|
+
const hasHooks = !!(profile.onInsert || profile.onUpdate || profile.onChange);
|
|
5290
|
+
let enrichedTotal = 0;
|
|
5291
|
+
let enrichSkipped = 0;
|
|
5292
|
+
let enrichRateLimited = 0;
|
|
5293
|
+
try {
|
|
5294
|
+
db = await openDatabase(platform);
|
|
5295
|
+
let sinceDate = null;
|
|
5296
|
+
if (options.fullRefresh) {
|
|
5297
|
+
sinceDate = null;
|
|
5298
|
+
} else if (options.since) {
|
|
5299
|
+
sinceDate = parseSince(options.since);
|
|
5300
|
+
} else if (!options.force && existingState?.lastSync) {
|
|
5301
|
+
sinceDate = new Date(existingState.lastSync);
|
|
5302
|
+
} else {
|
|
5303
|
+
sinceDate = /* @__PURE__ */ new Date();
|
|
5304
|
+
sinceDate.setDate(sinceDate.getDate() - DEFAULT_SINCE_DAYS);
|
|
5305
|
+
}
|
|
5306
|
+
const queryParams = { ...profile.queryParams };
|
|
5307
|
+
const bodyParams = { ...profile.body };
|
|
5308
|
+
const pageSize = profile.defaultLimit ?? 100;
|
|
5309
|
+
const limitLocation = profile.limitLocation || "query";
|
|
5310
|
+
let limitParam;
|
|
5311
|
+
if (profile.limitParam !== void 0) {
|
|
5312
|
+
limitParam = profile.limitParam;
|
|
5313
|
+
} else if (profile.pagination.type === "none") {
|
|
5314
|
+
limitParam = "";
|
|
5315
|
+
} else {
|
|
5316
|
+
limitParam = "limit";
|
|
5317
|
+
}
|
|
5318
|
+
if (limitParam) {
|
|
5319
|
+
if (limitLocation === "body") {
|
|
5320
|
+
bodyParams[limitParam] = pageSize;
|
|
5321
|
+
} else {
|
|
5322
|
+
queryParams[limitParam] = pageSize;
|
|
5323
|
+
}
|
|
5324
|
+
}
|
|
5325
|
+
if (sinceDate && profile.dateFilter) {
|
|
5326
|
+
const { param, format } = profile.dateFilter;
|
|
5327
|
+
let dateValue;
|
|
5328
|
+
if (format === "iso8601") {
|
|
5329
|
+
dateValue = sinceDate.toISOString();
|
|
5330
|
+
} else if (format === "unix") {
|
|
5331
|
+
dateValue = Math.floor(sinceDate.getTime() / 1e3);
|
|
5332
|
+
} else if (format === "date") {
|
|
5333
|
+
dateValue = sinceDate.toISOString().split("T")[0];
|
|
5334
|
+
}
|
|
5335
|
+
if (dateValue !== void 0) {
|
|
5336
|
+
queryParams[param] = dateValue;
|
|
5337
|
+
}
|
|
5338
|
+
}
|
|
5339
|
+
const actionDetails = await api.getActionDetails(profile.actionId);
|
|
5340
|
+
const maxPages = options.maxPages ?? Infinity;
|
|
5341
|
+
let tableCreated = tableExists(db, model);
|
|
5342
|
+
let currentPageQueryParams = { ...queryParams };
|
|
5343
|
+
let currentPageBodyParams = { ...bodyParams };
|
|
5344
|
+
let currentPageHeaders;
|
|
5345
|
+
let startPage = 0;
|
|
5346
|
+
if (!options.force && !options.fullRefresh && existingState?.status === "failed" && existingState.lastCursor != null && profile.pagination.passAs) {
|
|
5347
|
+
const { location, paramName } = parsePassAs(profile.pagination.passAs);
|
|
5348
|
+
if (location === "header") {
|
|
5349
|
+
currentPageHeaders = { ...currentPageHeaders, [paramName]: String(existingState.lastCursor) };
|
|
5350
|
+
} else if (location === "body") {
|
|
5351
|
+
currentPageBodyParams[paramName] = existingState.lastCursor;
|
|
5352
|
+
} else {
|
|
5353
|
+
currentPageQueryParams[paramName] = existingState.lastCursor;
|
|
5354
|
+
}
|
|
5355
|
+
startPage = existingState.pagesProcessed;
|
|
5356
|
+
if (!isAgentMode()) {
|
|
5357
|
+
process.stderr.write(
|
|
5358
|
+
`Resuming ${platform}/${model} from page ${startPage + 1} (cursor: ${String(existingState.lastCursor).slice(0, 40)})
|
|
5359
|
+
`
|
|
5360
|
+
);
|
|
5361
|
+
}
|
|
5362
|
+
}
|
|
5363
|
+
for (let page = startPage; page < maxPages + startPage; page++) {
|
|
5364
|
+
let responseData;
|
|
5365
|
+
let retries = 0;
|
|
5366
|
+
while (true) {
|
|
5367
|
+
try {
|
|
5368
|
+
const hasBody = Object.keys(currentPageBodyParams).length > 0;
|
|
5369
|
+
const result = await api.executePassthroughRequest({
|
|
5370
|
+
platform,
|
|
5371
|
+
actionId: profile.actionId,
|
|
5372
|
+
connectionKey: profile.connectionKey,
|
|
5373
|
+
pathVariables: profile.pathVars,
|
|
5374
|
+
queryParams: currentPageQueryParams,
|
|
5375
|
+
headers: currentPageHeaders,
|
|
5376
|
+
data: hasBody ? currentPageBodyParams : void 0
|
|
5377
|
+
}, actionDetails);
|
|
5378
|
+
responseData = result.responseData;
|
|
5379
|
+
break;
|
|
5380
|
+
} catch (err) {
|
|
5381
|
+
if (err instanceof ApiError) {
|
|
5382
|
+
const isRateLimited = err.status === 429;
|
|
5383
|
+
const isServerError = err.status >= 500 && err.status <= 504;
|
|
5384
|
+
if ((isRateLimited || isServerError) && retries < MAX_RETRIES_PER_PAGE) {
|
|
5385
|
+
const retryAfter = isRateLimited ? err.retryAfterSeconds ?? Math.min(30 * Math.pow(2, retries), 120) : Math.min(5 * Math.pow(2, retries), 30);
|
|
5386
|
+
const label = isRateLimited ? "Rate limited" : `Server error (${err.status})`;
|
|
5387
|
+
if (!isAgentMode()) {
|
|
5388
|
+
process.stderr.write(` ${label}. Waiting ${retryAfter}s before retry (${retries + 1}/${MAX_RETRIES_PER_PAGE})...
|
|
5389
|
+
`);
|
|
5390
|
+
}
|
|
5391
|
+
await sleep2(retryAfter * 1e3);
|
|
5392
|
+
retries++;
|
|
5393
|
+
continue;
|
|
5394
|
+
}
|
|
5395
|
+
if (err.status === 401 || err.status === 403) {
|
|
5396
|
+
throw new Error(`Authentication failed. Connection key may be expired. Run 'one add ${platform}' to refresh.`);
|
|
5397
|
+
}
|
|
5398
|
+
}
|
|
5399
|
+
if (isNetworkError(err) && retries < MAX_RETRIES_PER_PAGE) {
|
|
5400
|
+
const backoff = Math.min(2 * Math.pow(2, retries), 16);
|
|
5401
|
+
if (!isAgentMode()) {
|
|
5402
|
+
process.stderr.write(` Network error. Retrying in ${backoff}s (${retries + 1}/${MAX_RETRIES_PER_PAGE})...
|
|
5403
|
+
`);
|
|
5404
|
+
}
|
|
5405
|
+
await sleep2(backoff * 1e3);
|
|
5406
|
+
retries++;
|
|
5407
|
+
continue;
|
|
5408
|
+
}
|
|
5409
|
+
throw err;
|
|
5410
|
+
}
|
|
5411
|
+
}
|
|
5412
|
+
const records = getByDotPath(responseData, profile.resultsPath);
|
|
5413
|
+
if (!Array.isArray(records)) {
|
|
5414
|
+
const topKeys = typeof responseData === "object" && responseData !== null ? Object.keys(responseData) : [];
|
|
5415
|
+
throw new Error(
|
|
5416
|
+
`Could not find results at path '${profile.resultsPath}' in API response. Check your sync profile. Response keys: [${topKeys.join(", ")}]`
|
|
5417
|
+
);
|
|
5418
|
+
}
|
|
5419
|
+
if (records.length === 0 && page === 0) {
|
|
5420
|
+
if (!isAgentMode()) {
|
|
5421
|
+
process.stderr.write(`No records found for ${model} with the given filters.
|
|
5422
|
+
`);
|
|
5423
|
+
}
|
|
5424
|
+
break;
|
|
5425
|
+
}
|
|
5426
|
+
if (records.length === 0) break;
|
|
5427
|
+
if (options.dryRun) {
|
|
5428
|
+
pagesProcessed = 1;
|
|
5429
|
+
totalRecords = records.length;
|
|
5430
|
+
db.close();
|
|
5431
|
+
return {
|
|
5432
|
+
model,
|
|
5433
|
+
recordsSynced: totalRecords,
|
|
5434
|
+
pagesProcessed,
|
|
5435
|
+
duration: formatDuration(Date.now() - startTime),
|
|
5436
|
+
status: "dry-run"
|
|
5437
|
+
};
|
|
5438
|
+
}
|
|
5439
|
+
if (profile.transform) {
|
|
5440
|
+
const transformed = await transformRecords(profile.transform, records);
|
|
5441
|
+
if (transformed) {
|
|
5442
|
+
records.length = 0;
|
|
5443
|
+
for (const r of transformed) records.push(r);
|
|
5444
|
+
}
|
|
5445
|
+
}
|
|
5446
|
+
if (profile.exclude && profile.exclude.length > 0) {
|
|
5447
|
+
for (const record of records) {
|
|
5448
|
+
stripFields(record, profile.exclude);
|
|
5449
|
+
}
|
|
5450
|
+
}
|
|
5451
|
+
if (profile.identityKey) {
|
|
5452
|
+
for (const record of records) {
|
|
5453
|
+
const raw = getByDotPath(record, profile.identityKey);
|
|
5454
|
+
if (raw !== null && raw !== void 0) {
|
|
5455
|
+
record._identity = String(raw).toLowerCase().trim();
|
|
5456
|
+
}
|
|
5457
|
+
}
|
|
5458
|
+
}
|
|
5459
|
+
if (!tableCreated) {
|
|
5460
|
+
const firstRecord = records[0];
|
|
5461
|
+
ensureTable(db, model, firstRecord, profile.idField);
|
|
5462
|
+
tableCreated = true;
|
|
5463
|
+
}
|
|
5464
|
+
for (const record of records) {
|
|
5465
|
+
if (typeof record === "object" && record !== null) {
|
|
5466
|
+
evolveSchema(db, model, record);
|
|
5467
|
+
break;
|
|
5468
|
+
}
|
|
5469
|
+
}
|
|
5470
|
+
if (options.fullRefresh) {
|
|
5471
|
+
for (const rec of records) {
|
|
5472
|
+
const id = rec[profile.idField];
|
|
5473
|
+
if (typeof id === "string" || typeof id === "number") {
|
|
5474
|
+
seenIds.add(id);
|
|
5475
|
+
}
|
|
5476
|
+
}
|
|
5477
|
+
}
|
|
5478
|
+
let inserts = [];
|
|
5479
|
+
let updates = [];
|
|
5480
|
+
if (hasHooks) {
|
|
5481
|
+
const classified = classifyRecords(
|
|
5482
|
+
db,
|
|
5483
|
+
model,
|
|
5484
|
+
records,
|
|
5485
|
+
profile.idField,
|
|
5486
|
+
tableCreated
|
|
5487
|
+
);
|
|
5488
|
+
inserts = classified.inserts;
|
|
5489
|
+
updates = classified.updates;
|
|
5490
|
+
}
|
|
5491
|
+
const inserted = upsertRecords(
|
|
5492
|
+
db,
|
|
5493
|
+
model,
|
|
5494
|
+
records,
|
|
5495
|
+
profile.idField
|
|
5496
|
+
);
|
|
5497
|
+
totalRecords += inserted;
|
|
5498
|
+
pagesProcessed++;
|
|
5499
|
+
if (hasHooks) {
|
|
5500
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
5501
|
+
const buildEvents = (type, recs) => recs.map((record) => ({ type, platform, model, record, timestamp: now }));
|
|
5502
|
+
const insertEvents = buildEvents("insert", inserts);
|
|
5503
|
+
const updateEvents = buildEvents("update", updates);
|
|
5504
|
+
hooksInserted += inserts.length;
|
|
5505
|
+
hooksUpdated += updates.length;
|
|
5506
|
+
if (inserts.length > 0 && (profile.onInsert || profile.onChange)) {
|
|
5507
|
+
await fireHooks(profile.onInsert || profile.onChange, insertEvents);
|
|
5508
|
+
}
|
|
5509
|
+
if (updates.length > 0 && (profile.onUpdate || profile.onChange)) {
|
|
5510
|
+
await fireHooks(profile.onUpdate || profile.onChange, updateEvents);
|
|
5511
|
+
}
|
|
5512
|
+
}
|
|
5513
|
+
if (!isAgentMode()) {
|
|
5514
|
+
process.stderr.write(`Syncing ${platform}/${model}... page ${pagesProcessed} (${totalRecords} records)\r`);
|
|
5515
|
+
}
|
|
5516
|
+
const nextParams = getNextPageParams(
|
|
5517
|
+
responseData,
|
|
5518
|
+
profile.pagination,
|
|
5519
|
+
page,
|
|
5520
|
+
pageSize,
|
|
5521
|
+
records
|
|
5522
|
+
);
|
|
5523
|
+
const cursorBag = nextParams?.queryParams ?? nextParams?.bodyParams ?? nextParams?.headers;
|
|
5524
|
+
const cursorKeys = cursorBag ? Object.keys(cursorBag) : [];
|
|
5525
|
+
lastCursor = cursorKeys.length > 0 ? cursorBag[cursorKeys[0]] : null;
|
|
5526
|
+
updateModelState(platform, model, {
|
|
5527
|
+
totalRecords: countRecords(db, model),
|
|
5528
|
+
pagesProcessed,
|
|
5529
|
+
lastCursor,
|
|
5530
|
+
status: "syncing"
|
|
5531
|
+
});
|
|
5532
|
+
if (!nextParams) break;
|
|
5533
|
+
currentPageQueryParams = { ...queryParams, ...nextParams.queryParams };
|
|
5534
|
+
currentPageBodyParams = { ...bodyParams, ...nextParams.bodyParams };
|
|
5535
|
+
if (nextParams.headers) {
|
|
5536
|
+
currentPageHeaders = { ...currentPageHeaders, ...nextParams.headers };
|
|
5537
|
+
}
|
|
5538
|
+
}
|
|
5539
|
+
let deletedStale = 0;
|
|
5540
|
+
if (options.fullRefresh && pagesProcessed > 0 && tableCreated && seenIds.size > 0) {
|
|
5541
|
+
const safeTable = sanitizeTableName(model);
|
|
5542
|
+
const safeIdField = profile.idField.replace(/"/g, '""');
|
|
5543
|
+
const ids = Array.from(seenIds);
|
|
5544
|
+
const CHUNK = 500;
|
|
5545
|
+
const seenChunks = [];
|
|
5546
|
+
for (let i = 0; i < ids.length; i += CHUNK) {
|
|
5547
|
+
seenChunks.push(ids.slice(i, i + CHUNK));
|
|
5548
|
+
}
|
|
5549
|
+
db.exec(`CREATE TEMP TABLE IF NOT EXISTS _seen_ids (id)`);
|
|
5550
|
+
db.exec(`DELETE FROM _seen_ids`);
|
|
5551
|
+
const insertStmt = db.prepare(`INSERT INTO _seen_ids (id) VALUES (?)`);
|
|
5552
|
+
const tx = db.transaction((batch) => {
|
|
5553
|
+
for (const id of batch) insertStmt.run(id);
|
|
5554
|
+
});
|
|
5555
|
+
for (const chunk of seenChunks) tx(chunk);
|
|
5556
|
+
const delResult = db.prepare(
|
|
5557
|
+
`DELETE FROM "${safeTable}" WHERE "${safeIdField}" NOT IN (SELECT id FROM _seen_ids)`
|
|
5558
|
+
).run();
|
|
5559
|
+
deletedStale = delResult.changes;
|
|
5560
|
+
db.exec(`DROP TABLE IF EXISTS _seen_ids`);
|
|
5561
|
+
if (!isAgentMode() && deletedStale > 0) {
|
|
5562
|
+
process.stderr.write(`Removed ${deletedStale} stale record(s) no longer in source.
|
|
5563
|
+
`);
|
|
5564
|
+
}
|
|
5565
|
+
}
|
|
5566
|
+
if (pagesProcessed > 0 && tableCreated) {
|
|
5567
|
+
rebuildFtsIndex(db, model);
|
|
5568
|
+
}
|
|
5569
|
+
if (!isAgentMode() && pagesProcessed > 0) {
|
|
5570
|
+
process.stderr.write(`Syncing ${platform}/${model}... page ${pagesProcessed} (${totalRecords} records) done
|
|
5571
|
+
`);
|
|
5572
|
+
}
|
|
5573
|
+
let enrichResult = null;
|
|
5574
|
+
if (profile.enrich && tableCreated && !options.dryRun) {
|
|
5575
|
+
enrichResult = await enrichPhase(
|
|
5576
|
+
api,
|
|
5577
|
+
db,
|
|
5578
|
+
profile.enrich,
|
|
5579
|
+
model,
|
|
5580
|
+
profile.idField,
|
|
5581
|
+
profile.connectionKey,
|
|
5582
|
+
platform
|
|
5583
|
+
);
|
|
5584
|
+
enrichedTotal = enrichResult.enriched;
|
|
5585
|
+
enrichSkipped = enrichResult.skipped;
|
|
5586
|
+
enrichRateLimited = enrichResult.rateLimited;
|
|
5587
|
+
if (enrichResult.enriched > 0) {
|
|
5588
|
+
rebuildFtsIndex(db, model);
|
|
5589
|
+
}
|
|
5590
|
+
}
|
|
5591
|
+
const duration = formatDuration(Date.now() - startTime);
|
|
5592
|
+
const actualCount = countRecords(db, model);
|
|
5593
|
+
updateModelState(platform, model, {
|
|
5594
|
+
lastSync: (/* @__PURE__ */ new Date()).toISOString(),
|
|
5595
|
+
totalRecords: actualCount,
|
|
5596
|
+
pagesProcessed,
|
|
5597
|
+
lastCursor: null,
|
|
5598
|
+
since: sinceDate?.toISOString() ?? null,
|
|
5599
|
+
status: "idle"
|
|
5600
|
+
});
|
|
5601
|
+
db.close();
|
|
5602
|
+
if (lock) lock.release();
|
|
5603
|
+
return {
|
|
5604
|
+
model,
|
|
5605
|
+
recordsSynced: totalRecords,
|
|
5606
|
+
pagesProcessed,
|
|
5607
|
+
duration,
|
|
5608
|
+
status: "complete",
|
|
5609
|
+
deletedStale,
|
|
5610
|
+
...hasHooks ? { hooksInserted, hooksUpdated } : {},
|
|
5611
|
+
...profile.enrich ? { enriched: enrichedTotal, enrichSkipped, enrichRateLimited } : {}
|
|
5612
|
+
};
|
|
5613
|
+
} catch (err) {
|
|
5614
|
+
const failedCount = db ? countRecords(db, model) : existingState?.totalRecords ?? 0;
|
|
5615
|
+
updateModelState(platform, model, {
|
|
5616
|
+
status: "failed",
|
|
5617
|
+
totalRecords: failedCount,
|
|
5618
|
+
pagesProcessed,
|
|
5619
|
+
lastCursor
|
|
5620
|
+
});
|
|
5621
|
+
if (db) db.close();
|
|
5622
|
+
if (lock) lock.release();
|
|
5623
|
+
const rawMsg = err instanceof Error ? err.message : String(err);
|
|
5624
|
+
const shortMsg = truncate(rawMsg, 500);
|
|
5625
|
+
if (pagesProcessed > 0) {
|
|
5626
|
+
const resumeErr = new Error(
|
|
5627
|
+
`Sync interrupted after page ${pagesProcessed} (${totalRecords} records). Run again to resume. Error: ${shortMsg}`
|
|
5628
|
+
);
|
|
5629
|
+
resumeErr._recordsSynced = totalRecords;
|
|
5630
|
+
resumeErr._pagesProcessed = pagesProcessed;
|
|
5631
|
+
throw resumeErr;
|
|
5632
|
+
}
|
|
5633
|
+
const wrapped = new Error(shortMsg);
|
|
5634
|
+
wrapped._recordsSynced = totalRecords;
|
|
5635
|
+
wrapped._pagesProcessed = pagesProcessed;
|
|
5636
|
+
throw wrapped;
|
|
5637
|
+
}
|
|
5638
|
+
}
|
|
5639
|
+
|
|
5640
|
+
// src/lib/sync/test.ts
|
|
5641
|
+
function detectColumnType3(value) {
|
|
5642
|
+
if (value === null || value === void 0) return "TEXT";
|
|
5643
|
+
if (typeof value === "string") return "TEXT";
|
|
5644
|
+
if (typeof value === "boolean") return "INTEGER";
|
|
5645
|
+
if (typeof value === "number") return Number.isInteger(value) ? "INTEGER" : "REAL";
|
|
5646
|
+
if (typeof value === "object") return "TEXT (JSON)";
|
|
5647
|
+
return "TEXT";
|
|
5648
|
+
}
|
|
5649
|
+
async function testSyncProfile(api, profile) {
|
|
5650
|
+
const checks = [];
|
|
5651
|
+
const report = {
|
|
5652
|
+
platform: profile.platform,
|
|
5653
|
+
model: profile.model,
|
|
5654
|
+
ok: false,
|
|
5655
|
+
checks
|
|
5656
|
+
};
|
|
5657
|
+
const queryParams = { ...profile.queryParams };
|
|
5658
|
+
const bodyParams = { ...profile.body };
|
|
5659
|
+
const pageSize = profile.defaultLimit ?? 10;
|
|
5660
|
+
const limitLocation = profile.limitLocation || "query";
|
|
5661
|
+
let limitParam;
|
|
5662
|
+
if (profile.limitParam !== void 0) {
|
|
5663
|
+
limitParam = profile.limitParam;
|
|
5664
|
+
} else if (profile.pagination.type === "none") {
|
|
5665
|
+
limitParam = "";
|
|
5666
|
+
} else {
|
|
5667
|
+
limitParam = "limit";
|
|
5668
|
+
}
|
|
5669
|
+
if (limitParam) {
|
|
5670
|
+
if (limitLocation === "body") bodyParams[limitParam] = pageSize;
|
|
5671
|
+
else queryParams[limitParam] = pageSize;
|
|
5672
|
+
}
|
|
5673
|
+
let actionDetails;
|
|
5674
|
+
try {
|
|
5675
|
+
actionDetails = await api.getActionDetails(profile.actionId);
|
|
5676
|
+
checks.push({ name: "action resolves", ok: true });
|
|
5677
|
+
} catch (err) {
|
|
5678
|
+
checks.push({
|
|
5679
|
+
name: "action resolves",
|
|
5680
|
+
ok: false,
|
|
5681
|
+
detail: err instanceof Error ? err.message : String(err)
|
|
5682
|
+
});
|
|
5683
|
+
return report;
|
|
5684
|
+
}
|
|
5685
|
+
let responseData;
|
|
5686
|
+
try {
|
|
5687
|
+
const result = await api.executePassthroughRequest({
|
|
5688
|
+
platform: profile.platform,
|
|
5689
|
+
actionId: profile.actionId,
|
|
5690
|
+
connectionKey: profile.connectionKey,
|
|
5691
|
+
pathVariables: profile.pathVars,
|
|
5692
|
+
queryParams,
|
|
5693
|
+
data: Object.keys(bodyParams).length > 0 ? bodyParams : void 0
|
|
5694
|
+
}, actionDetails);
|
|
5695
|
+
responseData = result.responseData;
|
|
5696
|
+
checks.push({ name: "single-page fetch", ok: true });
|
|
5697
|
+
} catch (err) {
|
|
5698
|
+
const detail = err instanceof ApiError ? `HTTP ${err.status}: ${err.message}` : err instanceof Error ? err.message : String(err);
|
|
5699
|
+
checks.push({ name: "single-page fetch", ok: false, detail });
|
|
5700
|
+
return report;
|
|
5701
|
+
}
|
|
5702
|
+
let resolvedResultsPath = profile.resultsPath;
|
|
5703
|
+
let records = getByDotPath(responseData, resolvedResultsPath);
|
|
5704
|
+
if (!Array.isArray(records) && typeof responseData === "object" && responseData !== null) {
|
|
5705
|
+
const topObj = responseData;
|
|
5706
|
+
const arrayKey = Object.keys(topObj).find((k) => Array.isArray(topObj[k]));
|
|
5707
|
+
if (arrayKey) {
|
|
5708
|
+
resolvedResultsPath = arrayKey;
|
|
5709
|
+
records = topObj[arrayKey];
|
|
5710
|
+
report.autoFixed = report.autoFixed ?? {};
|
|
5711
|
+
report.autoFixed.resultsPath = arrayKey;
|
|
5712
|
+
checks.push({
|
|
5713
|
+
name: `resultsPath auto-discovered \u2192 "${arrayKey}"`,
|
|
5714
|
+
ok: true,
|
|
5715
|
+
detail: `Profile had "${profile.resultsPath}" which didn't resolve; found "${arrayKey}" in response`
|
|
5716
|
+
});
|
|
5717
|
+
}
|
|
5718
|
+
}
|
|
5719
|
+
if (!Array.isArray(records)) {
|
|
5720
|
+
const topKeys = typeof responseData === "object" && responseData !== null ? Object.keys(responseData) : [];
|
|
5721
|
+
checks.push({
|
|
5722
|
+
name: `resultsPath "${resolvedResultsPath}" \u2192 array`,
|
|
5723
|
+
ok: false,
|
|
5724
|
+
detail: `Not an array. Response keys: [${topKeys.join(", ")}]`
|
|
5725
|
+
});
|
|
5726
|
+
return report;
|
|
5727
|
+
}
|
|
5728
|
+
checks.push({
|
|
5729
|
+
name: `resultsPath "${resolvedResultsPath}" \u2192 array`,
|
|
5730
|
+
ok: true,
|
|
5731
|
+
detail: `${records.length} records`
|
|
5732
|
+
});
|
|
5733
|
+
if (records.length === 0) {
|
|
5734
|
+
checks.push({ name: "sample record available", ok: false, detail: "empty result set" });
|
|
5735
|
+
report.ok = checks.every((c) => c.ok === true || c.name === "sample record available");
|
|
5736
|
+
return report;
|
|
5737
|
+
}
|
|
5738
|
+
const first = records[0];
|
|
5739
|
+
let resolvedIdField = profile.idField;
|
|
5740
|
+
let idValue = first[resolvedIdField];
|
|
5741
|
+
if ((idValue === void 0 || idValue === null) && typeof first === "object") {
|
|
5742
|
+
const idCandidates = ["id", "_id", "uuid", "ID", "Id"];
|
|
5743
|
+
const found = idCandidates.find((c) => first[c] !== void 0 && first[c] !== null);
|
|
5744
|
+
if (found) {
|
|
5745
|
+
resolvedIdField = found;
|
|
5746
|
+
idValue = first[found];
|
|
5747
|
+
report.autoFixed = report.autoFixed ?? {};
|
|
5748
|
+
report.autoFixed.idField = found;
|
|
5749
|
+
checks.push({
|
|
5750
|
+
name: `idField auto-discovered \u2192 "${found}"`,
|
|
5751
|
+
ok: true,
|
|
5752
|
+
detail: `Profile had "${profile.idField}" which wasn't on the record; found "${found}"`
|
|
5753
|
+
});
|
|
5754
|
+
}
|
|
5755
|
+
}
|
|
5756
|
+
if (idValue === void 0 || idValue === null) {
|
|
5757
|
+
checks.push({
|
|
5758
|
+
name: `idField "${resolvedIdField}" present`,
|
|
5759
|
+
ok: false,
|
|
5760
|
+
detail: `Not found. Available fields: [${Object.keys(first).slice(0, 20).join(", ")}]`
|
|
5761
|
+
});
|
|
5762
|
+
} else {
|
|
5763
|
+
checks.push({
|
|
5764
|
+
name: `idField "${resolvedIdField}" present`,
|
|
5765
|
+
ok: true,
|
|
5766
|
+
detail: `sample value: ${String(idValue).slice(0, 60)}`
|
|
5767
|
+
});
|
|
5768
|
+
}
|
|
5769
|
+
const nextParams = getNextPageParams(responseData, profile.pagination, 0, pageSize, records);
|
|
5770
|
+
if (profile.pagination.type === "none") {
|
|
5771
|
+
checks.push({ name: "pagination type: none", ok: true });
|
|
5772
|
+
} else if (nextParams) {
|
|
5773
|
+
checks.push({
|
|
5774
|
+
name: `pagination "${profile.pagination.type}" \u2192 next page`,
|
|
5775
|
+
ok: true,
|
|
5776
|
+
detail: JSON.stringify(nextParams).slice(0, 120)
|
|
5777
|
+
});
|
|
5778
|
+
report.paginationPreview = nextParams;
|
|
5779
|
+
} else {
|
|
5780
|
+
checks.push({
|
|
5781
|
+
name: `pagination "${profile.pagination.type}"`,
|
|
5782
|
+
ok: true,
|
|
5783
|
+
detail: "no next page (single-page result or end-of-data)"
|
|
5784
|
+
});
|
|
5785
|
+
}
|
|
5786
|
+
report.detectedColumns = Object.entries(first).map(([name, value]) => ({
|
|
5787
|
+
name,
|
|
5788
|
+
type: detectColumnType3(value)
|
|
5789
|
+
}));
|
|
5790
|
+
report.sample = first;
|
|
5791
|
+
report.ok = checks.every((c) => c.ok);
|
|
5792
|
+
return report;
|
|
5793
|
+
}
|
|
5794
|
+
|
|
5795
|
+
// src/lib/sync/infer.ts
|
|
5796
|
+
var PAGINATION_PATTERNS = [
|
|
5797
|
+
// Notion-style: start_cursor + next_cursor in body (POST endpoints)
|
|
5798
|
+
{
|
|
5799
|
+
match: /start_cursor|next_cursor/i,
|
|
5800
|
+
config: {
|
|
5801
|
+
type: "cursor",
|
|
5802
|
+
nextPath: "next_cursor",
|
|
5803
|
+
passAs: "body:start_cursor",
|
|
5804
|
+
hasMorePath: "has_more"
|
|
5805
|
+
},
|
|
5806
|
+
label: "Notion-style body cursor pagination (start_cursor/next_cursor)"
|
|
5807
|
+
},
|
|
5808
|
+
// Stripe-style: starting_after cursor returning has_more (checked AFTER Notion so we don't match has_more alone)
|
|
5809
|
+
{
|
|
5810
|
+
match: /starting_after/i,
|
|
5811
|
+
config: {
|
|
5812
|
+
type: "id",
|
|
5813
|
+
passAs: "query:starting_after",
|
|
5814
|
+
hasMorePath: "has_more",
|
|
5815
|
+
idField: "id"
|
|
5816
|
+
},
|
|
5817
|
+
label: "Stripe-style id pagination (starting_after + has_more)"
|
|
5818
|
+
},
|
|
5819
|
+
// Shopify/GitHub Link-header cursor
|
|
5820
|
+
{
|
|
5821
|
+
match: /link:\s*<[^>]+>;\s*rel="next"|page_info=/i,
|
|
5822
|
+
config: { type: "link", nextPath: "headers.link", passAs: "query:page_info" },
|
|
5823
|
+
label: "Link header cursor pagination"
|
|
5824
|
+
},
|
|
5825
|
+
// HubSpot / Google: next page token
|
|
5826
|
+
{
|
|
5827
|
+
match: /next_page_token|nextPageToken|paging\.next\.after/i,
|
|
5828
|
+
config: {
|
|
5829
|
+
type: "token",
|
|
5830
|
+
nextPath: "paging.next.after",
|
|
5831
|
+
passAs: "query:after"
|
|
5832
|
+
},
|
|
5833
|
+
label: "token/after pagination (HubSpot/Google style)"
|
|
5834
|
+
},
|
|
5835
|
+
// Generic cursor
|
|
5836
|
+
{
|
|
5837
|
+
match: /next_cursor|nextCursor|cursor/i,
|
|
5838
|
+
config: { type: "cursor", nextPath: "next_cursor", passAs: "query:cursor" },
|
|
5839
|
+
label: "generic cursor pagination"
|
|
5840
|
+
},
|
|
5841
|
+
// Offset/limit
|
|
5842
|
+
{
|
|
5843
|
+
match: /offset.*limit|page.*per_page/i,
|
|
5844
|
+
config: { type: "offset", passAs: "query:offset" },
|
|
5845
|
+
label: "offset/limit pagination"
|
|
5846
|
+
}
|
|
5847
|
+
];
|
|
5848
|
+
function inferResultsPath(knowledge, modelName, platform) {
|
|
5849
|
+
const candidates = ["data", "results", "items", "records", "rows", "entries"];
|
|
5850
|
+
for (const key of candidates) {
|
|
5851
|
+
const re = new RegExp(`"${key}"\\s*:\\s*\\[`, "i");
|
|
5852
|
+
if (re.test(knowledge)) return key;
|
|
5853
|
+
}
|
|
5854
|
+
if (modelName) {
|
|
5855
|
+
const namesToTry = /* @__PURE__ */ new Set();
|
|
5856
|
+
namesToTry.add(modelName);
|
|
5857
|
+
namesToTry.add(modelName.toLowerCase());
|
|
5858
|
+
if (platform) {
|
|
5859
|
+
const lower = modelName.toLowerCase();
|
|
5860
|
+
const platLower = platform.toLowerCase().replace(/-/g, "");
|
|
5861
|
+
if (lower.startsWith(platLower)) {
|
|
5862
|
+
const stripped = modelName.slice(platLower.length);
|
|
5863
|
+
if (stripped.length > 0) {
|
|
5864
|
+
namesToTry.add(stripped[0].toLowerCase() + stripped.slice(1));
|
|
5865
|
+
}
|
|
5866
|
+
}
|
|
5867
|
+
}
|
|
5868
|
+
if (modelName.endsWith("ies")) {
|
|
5869
|
+
namesToTry.add(modelName.slice(0, -3) + "y");
|
|
5870
|
+
} else if (modelName.endsWith("s")) {
|
|
5871
|
+
namesToTry.add(modelName.slice(0, -1));
|
|
5872
|
+
} else {
|
|
5873
|
+
namesToTry.add(modelName + "s");
|
|
5874
|
+
}
|
|
5875
|
+
for (const name of namesToTry) {
|
|
5876
|
+
const reJson = new RegExp(`"${name}"\\s*:\\s*\\[`, "i");
|
|
5877
|
+
const reProse = new RegExp(`\\b${name}\\b.*\\barray\\b|\\barray\\b.*\\b${name}\\b`, "i");
|
|
5878
|
+
if (reJson.test(knowledge) || reProse.test(knowledge)) return name;
|
|
5879
|
+
}
|
|
5880
|
+
}
|
|
5881
|
+
return void 0;
|
|
5882
|
+
}
|
|
5883
|
+
function inferPathVars(knowledge) {
|
|
5884
|
+
const vars = {};
|
|
5885
|
+
const urlMatches = knowledge.matchAll(/\{\{?([a-zA-Z_][a-zA-Z0-9_]*)\}?\}/g);
|
|
5886
|
+
for (const m of urlMatches) {
|
|
5887
|
+
const name = m[1];
|
|
5888
|
+
if (["payload", "timestamp", "eventType", "connectionId", "relayEventId"].includes(name)) continue;
|
|
5889
|
+
if (isExcludedPathVar(name)) continue;
|
|
5890
|
+
vars[name] = suggestDefault(name);
|
|
5891
|
+
}
|
|
5892
|
+
if (Object.keys(vars).length === 0) return void 0;
|
|
5893
|
+
return vars;
|
|
5894
|
+
}
|
|
5895
|
+
var INTERNAL_PATH_VARS = /* @__PURE__ */ new Set([
|
|
5896
|
+
"internal_signing_key",
|
|
5897
|
+
"signing_key",
|
|
5898
|
+
"api_key",
|
|
5899
|
+
"apikey",
|
|
5900
|
+
"secret",
|
|
5901
|
+
"token",
|
|
5902
|
+
"access_token",
|
|
5903
|
+
"refresh_token"
|
|
5904
|
+
]);
|
|
5905
|
+
var RECORD_LEVEL_PATH_VARS = /* @__PURE__ */ new Set([
|
|
5906
|
+
"record_id",
|
|
5907
|
+
"recordid",
|
|
5908
|
+
"id",
|
|
5909
|
+
"itemid",
|
|
5910
|
+
"item_id",
|
|
5911
|
+
"pageid",
|
|
5912
|
+
"page_id",
|
|
5913
|
+
"objectid",
|
|
5914
|
+
"object_id",
|
|
5915
|
+
"entryid",
|
|
5916
|
+
"entry_id",
|
|
5917
|
+
"resourceid",
|
|
5918
|
+
"resource_id"
|
|
5919
|
+
]);
|
|
5920
|
+
function suggestDefault(varName) {
|
|
5921
|
+
const lower = varName.toLowerCase();
|
|
5922
|
+
if (lower === "calendarid") return "primary";
|
|
5923
|
+
if (lower === "userid" || lower === "user_id") return "me";
|
|
5924
|
+
if (lower === "accountid" || lower === "account_id") return "me";
|
|
5925
|
+
return "FILL_IN";
|
|
5926
|
+
}
|
|
5927
|
+
function isExcludedPathVar(varName) {
|
|
5928
|
+
const lower = varName.toLowerCase();
|
|
5929
|
+
return INTERNAL_PATH_VARS.has(lower) || RECORD_LEVEL_PATH_VARS.has(lower);
|
|
5930
|
+
}
|
|
5931
|
+
function inferIdField(knowledge) {
|
|
5932
|
+
if (/"id"\s*:/.test(knowledge)) return "id";
|
|
5933
|
+
if (/\b_id\b/.test(knowledge)) return "_id";
|
|
5934
|
+
if (/\buuid\b/i.test(knowledge)) return "uuid";
|
|
5935
|
+
return void 0;
|
|
5936
|
+
}
|
|
5937
|
+
function inferDateFilter(knowledge) {
|
|
5938
|
+
const candidates = ["updated_since", "updatedSince", "modified_since", "since", "updated_after", "created_after"];
|
|
5939
|
+
for (const c of candidates) {
|
|
5940
|
+
if (new RegExp(`\\b${c}\\b`, "i").test(knowledge)) return c;
|
|
5941
|
+
}
|
|
5942
|
+
return void 0;
|
|
5943
|
+
}
|
|
5944
|
+
function inferProfileFromKnowledge(knowledge, modelName, platform) {
|
|
5945
|
+
const hints = { reasoning: [] };
|
|
5946
|
+
if (!knowledge) {
|
|
5947
|
+
hints.reasoning.push("No knowledge available; all fields left as FILL_IN.");
|
|
5948
|
+
return hints;
|
|
5949
|
+
}
|
|
5950
|
+
for (const pattern of PAGINATION_PATTERNS) {
|
|
5951
|
+
if (pattern.match.test(knowledge)) {
|
|
5952
|
+
hints.pagination = { ...pattern.config };
|
|
5953
|
+
if (hints.pagination.type === "offset") {
|
|
5954
|
+
delete hints.pagination.nextPath;
|
|
5955
|
+
delete hints.pagination.hasMorePath;
|
|
5956
|
+
} else if (hints.pagination.type === "none") {
|
|
5957
|
+
delete hints.pagination.nextPath;
|
|
5958
|
+
delete hints.pagination.passAs;
|
|
5959
|
+
delete hints.pagination.hasMorePath;
|
|
5960
|
+
}
|
|
5961
|
+
hints.reasoning.push(`Pagination: ${pattern.label}`);
|
|
5962
|
+
break;
|
|
5963
|
+
}
|
|
5964
|
+
}
|
|
5965
|
+
const resultsPath = inferResultsPath(knowledge, modelName, platform);
|
|
5966
|
+
if (resultsPath) {
|
|
5967
|
+
hints.resultsPath = resultsPath;
|
|
5968
|
+
hints.reasoning.push(`resultsPath: "${resultsPath}" (found in response schema)`);
|
|
5969
|
+
}
|
|
5970
|
+
const idField = inferIdField(knowledge);
|
|
5971
|
+
if (idField) {
|
|
5972
|
+
hints.idField = idField;
|
|
5973
|
+
hints.reasoning.push(`idField: "${idField}"`);
|
|
5974
|
+
}
|
|
5975
|
+
const dateFilter = inferDateFilter(knowledge);
|
|
5976
|
+
if (dateFilter) {
|
|
5977
|
+
hints.dateFilterParam = dateFilter;
|
|
5978
|
+
hints.reasoning.push(`dateFilter candidate: "${dateFilter}" (for incremental sync)`);
|
|
5979
|
+
}
|
|
5980
|
+
const pathVars = inferPathVars(knowledge);
|
|
5981
|
+
if (pathVars) {
|
|
5982
|
+
hints.pathVars = pathVars;
|
|
5983
|
+
const varList = Object.entries(pathVars).map(([k, v]) => v === "FILL_IN" ? k : `${k}="${v}"`).join(", ");
|
|
5984
|
+
hints.reasoning.push(`pathVars: {${varList}} (extracted from URL template)`);
|
|
5985
|
+
}
|
|
5986
|
+
const isPost = /\bPOST\b/.test(knowledge);
|
|
5987
|
+
if (isPost) {
|
|
5988
|
+
const bodyLimitMatch = knowledge.match(/\b(page_size|pageSize|limit|max_results|maxResults)\b/);
|
|
5989
|
+
if (bodyLimitMatch) {
|
|
5990
|
+
hints.limitLocation = "body";
|
|
5991
|
+
hints.limitParam = bodyLimitMatch[1];
|
|
5992
|
+
hints.reasoning.push(
|
|
5993
|
+
`limitLocation: "body" (POST endpoint \u2014 page size "${bodyLimitMatch[1]}" goes in request body)`
|
|
5994
|
+
);
|
|
5995
|
+
if (hints.pagination && hints.pagination.passAs && hints.pagination.passAs.startsWith("query:")) {
|
|
5996
|
+
const paramName = hints.pagination.passAs.slice("query:".length);
|
|
5997
|
+
hints.pagination.passAs = `body:${paramName}`;
|
|
5998
|
+
hints.reasoning.push(`adjusted pagination passAs to "body:${paramName}" (POST endpoint)`);
|
|
5999
|
+
}
|
|
6000
|
+
}
|
|
6001
|
+
}
|
|
6002
|
+
if (hints.reasoning.length === 0) {
|
|
6003
|
+
hints.reasoning.push("Could not infer any fields from knowledge \u2014 fill template manually.");
|
|
6004
|
+
}
|
|
6005
|
+
return hints;
|
|
6006
|
+
}
|
|
6007
|
+
|
|
6008
|
+
// src/lib/sync/builtin-profiles.ts
|
|
6009
|
+
import fs13 from "fs";
|
|
6010
|
+
import path13 from "path";
|
|
6011
|
+
import { fileURLToPath as fileURLToPath3 } from "url";
|
|
6012
|
+
function getProfilesDir() {
|
|
6013
|
+
const thisFile = fileURLToPath3(import.meta.url);
|
|
6014
|
+
const thisDir = path13.dirname(thisFile);
|
|
6015
|
+
for (let i = 1; i <= 4; i++) {
|
|
6016
|
+
const candidate = path13.resolve(thisDir, ...Array(i).fill(".."), "profiles");
|
|
6017
|
+
if (fs13.existsSync(candidate)) return candidate;
|
|
6018
|
+
}
|
|
6019
|
+
return "";
|
|
6020
|
+
}
|
|
6021
|
+
function loadBuiltinProfile(platform, model) {
|
|
6022
|
+
const dir = getProfilesDir();
|
|
6023
|
+
if (!dir) return null;
|
|
6024
|
+
const filePath = path13.join(dir, platform, `${model}.json`);
|
|
6025
|
+
try {
|
|
6026
|
+
if (!fs13.existsSync(filePath)) return null;
|
|
6027
|
+
const raw = fs13.readFileSync(filePath, "utf-8");
|
|
6028
|
+
return JSON.parse(raw);
|
|
6029
|
+
} catch {
|
|
6030
|
+
return null;
|
|
6031
|
+
}
|
|
6032
|
+
}
|
|
6033
|
+
function listBuiltinProfiles(platform) {
|
|
6034
|
+
const dir = getProfilesDir();
|
|
6035
|
+
if (!dir) return [];
|
|
6036
|
+
const profiles = [];
|
|
6037
|
+
try {
|
|
6038
|
+
const platforms = platform ? [platform] : fs13.readdirSync(dir).filter((f) => {
|
|
6039
|
+
try {
|
|
6040
|
+
return fs13.statSync(path13.join(dir, f)).isDirectory();
|
|
6041
|
+
} catch {
|
|
6042
|
+
return false;
|
|
6043
|
+
}
|
|
6044
|
+
});
|
|
6045
|
+
for (const plat of platforms) {
|
|
6046
|
+
const platDir = path13.join(dir, plat);
|
|
6047
|
+
if (!fs13.existsSync(platDir)) continue;
|
|
6048
|
+
const files = fs13.readdirSync(platDir).filter((f) => f.endsWith(".json"));
|
|
6049
|
+
for (const file of files) {
|
|
6050
|
+
try {
|
|
6051
|
+
const raw = fs13.readFileSync(path13.join(platDir, file), "utf-8");
|
|
6052
|
+
const profile = JSON.parse(raw);
|
|
6053
|
+
profiles.push(profile);
|
|
6054
|
+
} catch {
|
|
6055
|
+
}
|
|
6056
|
+
}
|
|
6057
|
+
}
|
|
6058
|
+
} catch {
|
|
6059
|
+
}
|
|
6060
|
+
return profiles;
|
|
6061
|
+
}
|
|
6062
|
+
|
|
6063
|
+
// src/lib/sync/schedule.ts
|
|
6064
|
+
import { spawnSync as spawnSync2 } from "child_process";
|
|
6065
|
+
import fs15 from "fs";
|
|
6066
|
+
import os7 from "os";
|
|
6067
|
+
import path15 from "path";
|
|
6068
|
+
|
|
6069
|
+
// src/lib/sync/schedule-registry.ts
|
|
6070
|
+
import fs14 from "fs";
|
|
6071
|
+
import os6 from "os";
|
|
6072
|
+
import path14 from "path";
|
|
6073
|
+
var REGISTRY_DIR = path14.join(os6.homedir(), ".one", "sync");
|
|
6074
|
+
var REGISTRY_FILE = path14.join(REGISTRY_DIR, "schedules.json");
|
|
6075
|
+
function readRaw() {
|
|
6076
|
+
try {
|
|
6077
|
+
if (!fs14.existsSync(REGISTRY_FILE)) return { schedules: [] };
|
|
6078
|
+
const raw = fs14.readFileSync(REGISTRY_FILE, "utf-8");
|
|
6079
|
+
const parsed = JSON.parse(raw);
|
|
6080
|
+
if (!parsed || !Array.isArray(parsed.schedules)) return { schedules: [] };
|
|
6081
|
+
return parsed;
|
|
6082
|
+
} catch {
|
|
6083
|
+
return { schedules: [] };
|
|
6084
|
+
}
|
|
6085
|
+
}
|
|
6086
|
+
function writeRaw(file) {
|
|
6087
|
+
fs14.mkdirSync(REGISTRY_DIR, { recursive: true });
|
|
6088
|
+
const tmp = REGISTRY_FILE + ".tmp";
|
|
6089
|
+
fs14.writeFileSync(tmp, JSON.stringify(file, null, 2));
|
|
6090
|
+
fs14.renameSync(tmp, REGISTRY_FILE);
|
|
6091
|
+
}
|
|
6092
|
+
function makeScheduleId(platform, cwd) {
|
|
6093
|
+
const slug = path14.basename(cwd).replace(/[^a-zA-Z0-9_-]/g, "-").toLowerCase();
|
|
6094
|
+
return `${platform}-${slug}`;
|
|
6095
|
+
}
|
|
6096
|
+
function listRegistered() {
|
|
6097
|
+
return readRaw().schedules;
|
|
6098
|
+
}
|
|
6099
|
+
function getRegistered(id) {
|
|
6100
|
+
return readRaw().schedules.find((s) => s.id === id);
|
|
6101
|
+
}
|
|
6102
|
+
function findByPlatform(platform, cwd) {
|
|
6103
|
+
const all = readRaw().schedules;
|
|
6104
|
+
return all.filter((s) => s.platform === platform && (cwd ? s.cwd === cwd : true));
|
|
6105
|
+
}
|
|
6106
|
+
function upsertRegistered(entry) {
|
|
6107
|
+
const file = readRaw();
|
|
6108
|
+
const idx = file.schedules.findIndex((s) => s.id === entry.id);
|
|
6109
|
+
if (idx >= 0) {
|
|
6110
|
+
file.schedules[idx] = entry;
|
|
6111
|
+
} else {
|
|
6112
|
+
file.schedules.push(entry);
|
|
6113
|
+
}
|
|
6114
|
+
writeRaw(file);
|
|
6115
|
+
}
|
|
6116
|
+
function removeRegistered(id) {
|
|
6117
|
+
const file = readRaw();
|
|
6118
|
+
const before = file.schedules.length;
|
|
6119
|
+
file.schedules = file.schedules.filter((s) => s.id !== id);
|
|
6120
|
+
if (file.schedules.length === before) return false;
|
|
6121
|
+
writeRaw(file);
|
|
6122
|
+
return true;
|
|
6123
|
+
}
|
|
6124
|
+
|
|
6125
|
+
// src/lib/sync/schedule.ts
|
|
6126
|
+
var MARKER = "# one-sync";
|
|
6127
|
+
var LOG_DIR_REL = path15.join(".one", "sync", "logs");
|
|
6128
|
+
function durationToCron(every) {
|
|
6129
|
+
const match = every.match(/^(\d+)([mhd])$/);
|
|
6130
|
+
if (!match) return null;
|
|
6131
|
+
const amount = parseInt(match[1], 10);
|
|
6132
|
+
const unit = match[2];
|
|
6133
|
+
if (unit === "m") {
|
|
6134
|
+
if (amount < 1 || amount > 59 || 60 % amount !== 0) return null;
|
|
6135
|
+
return `*/${amount} * * * *`;
|
|
6136
|
+
}
|
|
6137
|
+
if (unit === "h") {
|
|
6138
|
+
if (amount < 1 || amount > 23 || 24 % amount !== 0) return null;
|
|
6139
|
+
return `0 */${amount} * * *`;
|
|
6140
|
+
}
|
|
6141
|
+
if (unit === "d") {
|
|
6142
|
+
if (amount !== 1) return null;
|
|
6143
|
+
return "0 0 * * *";
|
|
6144
|
+
}
|
|
6145
|
+
return null;
|
|
6146
|
+
}
|
|
6147
|
+
function cronExprToDuration(expr) {
|
|
6148
|
+
const parts = expr.trim().split(/\s+/);
|
|
6149
|
+
if (parts.length !== 5) return null;
|
|
6150
|
+
const [min, hour, dom, mon, dow] = parts;
|
|
6151
|
+
if (dom !== "*" || mon !== "*" || dow !== "*") return null;
|
|
6152
|
+
const mm = min.match(/^\*\/(\d+)$/);
|
|
6153
|
+
if (mm && hour === "*") return `${mm[1]}m`;
|
|
6154
|
+
const hm = hour.match(/^\*\/(\d+)$/);
|
|
6155
|
+
if (hm && min === "0") return `${hm[1]}h`;
|
|
6156
|
+
if (min === "0" && hour === "0") return "1d";
|
|
6157
|
+
return null;
|
|
6158
|
+
}
|
|
6159
|
+
function isWindows() {
|
|
6160
|
+
return os7.platform() === "win32";
|
|
6161
|
+
}
|
|
6162
|
+
function resolveOneBinary() {
|
|
6163
|
+
try {
|
|
6164
|
+
const entry = process.argv[1];
|
|
6165
|
+
if (entry && fs15.existsSync(entry)) {
|
|
6166
|
+
return fs15.realpathSync(entry);
|
|
6167
|
+
}
|
|
6168
|
+
} catch {
|
|
6169
|
+
}
|
|
6170
|
+
return "one";
|
|
6171
|
+
}
|
|
6172
|
+
function readCrontab() {
|
|
6173
|
+
try {
|
|
6174
|
+
const result = spawnSync2("crontab", ["-l"], { encoding: "utf-8" });
|
|
6175
|
+
if (result.status !== 0) return "";
|
|
6176
|
+
return result.stdout || "";
|
|
6177
|
+
} catch {
|
|
6178
|
+
return "";
|
|
6179
|
+
}
|
|
6180
|
+
}
|
|
6181
|
+
function writeCrontab(content) {
|
|
6182
|
+
const result = spawnSync2("crontab", ["-"], { input: content, encoding: "utf-8" });
|
|
6183
|
+
if (result.status !== 0) {
|
|
6184
|
+
const stderr = result.stderr || "";
|
|
6185
|
+
if (stderr.includes("Operation not permitted")) {
|
|
6186
|
+
throw new Error(
|
|
6187
|
+
"crontab write was blocked by macOS privacy protection. Grant Full Disk Access to your terminal app in System Settings \u2192 Privacy & Security \u2192 Full Disk Access, then retry."
|
|
6188
|
+
);
|
|
6189
|
+
}
|
|
6190
|
+
throw new Error(`Failed to write crontab: ${stderr || "unknown error"}`);
|
|
6191
|
+
}
|
|
6192
|
+
}
|
|
6193
|
+
function buildCronLine(entry) {
|
|
6194
|
+
const modelsArg = entry.models && entry.models.length > 0 ? ` --models ${entry.models.join(",")}` : "";
|
|
6195
|
+
const command = `cd ${JSON.stringify(entry.cwd)} && ${JSON.stringify(entry.nodeBin)} ${JSON.stringify(entry.cliBin)} sync run ${entry.platform}${modelsArg} >> ${JSON.stringify(entry.logFile)} 2>&1`;
|
|
6196
|
+
return `${entry.cronExpr} ${command} ${MARKER}:${entry.id}`;
|
|
6197
|
+
}
|
|
6198
|
+
function crontabHasId(crontab, id, legacyPlatform) {
|
|
6199
|
+
const lines = crontab.split("\n");
|
|
6200
|
+
return lines.some(
|
|
6201
|
+
(l) => l.includes(`${MARKER}:${id}`) || (legacyPlatform ? l.includes(`${MARKER}:${legacyPlatform}`) : false)
|
|
6202
|
+
);
|
|
6203
|
+
}
|
|
6204
|
+
function removeCronLines(crontab, id, legacyPlatform) {
|
|
6205
|
+
const lines = crontab.split("\n");
|
|
6206
|
+
const filtered = lines.filter(
|
|
6207
|
+
(l) => !l.includes(`${MARKER}:${id}`) && !(legacyPlatform && l.includes(`${MARKER}:${legacyPlatform}`) && l.includes(process.cwd()))
|
|
6208
|
+
);
|
|
6209
|
+
return filtered.filter((l) => l.length > 0).join("\n") + "\n";
|
|
6210
|
+
}
|
|
6211
|
+
function migrateLegacyCronEntries() {
|
|
6212
|
+
const crontab = readCrontab();
|
|
6213
|
+
if (!crontab.includes(MARKER)) return;
|
|
6214
|
+
const registered = listRegistered();
|
|
6215
|
+
const registeredIds = new Set(registered.map((s) => s.id));
|
|
6216
|
+
for (const line of crontab.split("\n")) {
|
|
6217
|
+
if (!line.includes(MARKER)) continue;
|
|
6218
|
+
const tagMatch = line.match(new RegExp(`${MARKER}:(\\S+)\\s*$`));
|
|
6219
|
+
if (!tagMatch) continue;
|
|
6220
|
+
const tag = tagMatch[1];
|
|
6221
|
+
if (registeredIds.has(tag)) continue;
|
|
6222
|
+
const parseMatch = line.match(/^(\S+\s+\S+\s+\S+\s+\S+\s+\S+)\s+(.+?)\s+#/);
|
|
6223
|
+
if (!parseMatch) continue;
|
|
6224
|
+
const cronExpr = parseMatch[1];
|
|
6225
|
+
const command = parseMatch[2];
|
|
6226
|
+
const cwdMatch = command.match(/^cd\s+"([^"]+)"/);
|
|
6227
|
+
const cwd = cwdMatch ? cwdMatch[1] : process.cwd();
|
|
6228
|
+
const twoPath = command.match(/"([^"]+)"\s+"([^"]+)"\s+sync\s+run\s+(\S+)/);
|
|
6229
|
+
const onePath = command.match(/&&\s+"([^"]+)"\s+sync\s+run\s+(\S+)/);
|
|
6230
|
+
let nodeBin = process.execPath;
|
|
6231
|
+
let cliBin = resolveOneBinary();
|
|
6232
|
+
let platform = tag;
|
|
6233
|
+
if (twoPath) {
|
|
6234
|
+
nodeBin = twoPath[1];
|
|
6235
|
+
cliBin = twoPath[2];
|
|
6236
|
+
platform = twoPath[3];
|
|
6237
|
+
} else if (onePath) {
|
|
6238
|
+
cliBin = onePath[1];
|
|
6239
|
+
platform = onePath[2];
|
|
6240
|
+
}
|
|
6241
|
+
const modelsMatch = command.match(/--models\s+(\S+)/);
|
|
6242
|
+
const models = modelsMatch ? modelsMatch[1].split(",") : void 0;
|
|
6243
|
+
const logMatch = command.match(/>>\s+"([^"]+)"/);
|
|
6244
|
+
const logFile = logMatch ? logMatch[1] : path15.resolve(cwd, LOG_DIR_REL, `${platform}.log`);
|
|
6245
|
+
const id = makeScheduleId(platform, cwd);
|
|
6246
|
+
if (registeredIds.has(id)) continue;
|
|
6247
|
+
upsertRegistered({
|
|
6248
|
+
id,
|
|
6249
|
+
platform,
|
|
6250
|
+
models,
|
|
6251
|
+
every: cronExprToDuration(cronExpr) ?? cronExpr,
|
|
6252
|
+
cronExpr,
|
|
6253
|
+
cwd,
|
|
6254
|
+
nodeBin,
|
|
6255
|
+
cliBin,
|
|
6256
|
+
logFile,
|
|
6257
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
6258
|
+
});
|
|
6259
|
+
registeredIds.add(id);
|
|
6260
|
+
}
|
|
6261
|
+
}
|
|
6262
|
+
function addSchedule(opts) {
|
|
6263
|
+
if (isWindows()) {
|
|
6264
|
+
throw new Error(
|
|
6265
|
+
"Scheduling via `one sync schedule` is not supported on Windows yet. Use Task Scheduler manually: create a task that runs `one sync run " + opts.platform + "` on your interval."
|
|
6266
|
+
);
|
|
6267
|
+
}
|
|
6268
|
+
const cronExpr = durationToCron(opts.every);
|
|
6269
|
+
if (!cronExpr) {
|
|
6270
|
+
throw new Error(
|
|
6271
|
+
`Invalid --every value "${opts.every}". Supported: <n>m (must divide 60), <n>h (must divide 24), or 1d. Examples: 15m, 30m, 1h, 6h, 12h, 1d`
|
|
6272
|
+
);
|
|
6273
|
+
}
|
|
6274
|
+
migrateLegacyCronEntries();
|
|
6275
|
+
const cwd = process.cwd();
|
|
6276
|
+
const id = makeScheduleId(opts.platform, cwd);
|
|
6277
|
+
const replaced = getRegistered(id) !== void 0;
|
|
6278
|
+
const logDir = path15.join(cwd, LOG_DIR_REL);
|
|
6279
|
+
fs15.mkdirSync(logDir, { recursive: true });
|
|
6280
|
+
const logFile = path15.join(logDir, `${opts.platform}.log`);
|
|
6281
|
+
const entry = {
|
|
6282
|
+
id,
|
|
6283
|
+
platform: opts.platform,
|
|
6284
|
+
models: opts.models,
|
|
6285
|
+
every: opts.every,
|
|
6286
|
+
cronExpr,
|
|
6287
|
+
cwd,
|
|
6288
|
+
nodeBin: process.execPath,
|
|
6289
|
+
cliBin: resolveOneBinary(),
|
|
6290
|
+
logFile,
|
|
6291
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
6292
|
+
};
|
|
6293
|
+
const current = readCrontab();
|
|
6294
|
+
const cleaned = removeCronLines(current, id, opts.platform);
|
|
6295
|
+
const nextLine = buildCronLine(entry);
|
|
6296
|
+
const next = (cleaned.trimEnd() + "\n" + nextLine + "\n").replace(/^\n+/, "");
|
|
6297
|
+
writeCrontab(next);
|
|
6298
|
+
upsertRegistered(entry);
|
|
6299
|
+
return { entry, replaced };
|
|
6300
|
+
}
|
|
6301
|
+
function listSchedules() {
|
|
6302
|
+
migrateLegacyCronEntries();
|
|
6303
|
+
const registered = listRegistered();
|
|
6304
|
+
const crontab = readCrontab();
|
|
6305
|
+
return registered.map((entry) => ({
|
|
6306
|
+
...entry,
|
|
6307
|
+
cronInstalled: crontabHasId(crontab, entry.id, entry.platform)
|
|
6308
|
+
}));
|
|
6309
|
+
}
|
|
6310
|
+
function removeSchedule(idOrPlatform, options) {
|
|
6311
|
+
migrateLegacyCronEntries();
|
|
6312
|
+
const byId = getRegistered(idOrPlatform);
|
|
6313
|
+
let toRemove;
|
|
6314
|
+
if (byId) {
|
|
6315
|
+
toRemove = [byId];
|
|
6316
|
+
} else {
|
|
6317
|
+
const cwd = options?.allProjects ? void 0 : process.cwd();
|
|
6318
|
+
toRemove = findByPlatform(idOrPlatform, cwd);
|
|
6319
|
+
}
|
|
6320
|
+
if (toRemove.length === 0) {
|
|
6321
|
+
return { removed: [], notFound: true };
|
|
6322
|
+
}
|
|
6323
|
+
let crontab = readCrontab();
|
|
6324
|
+
for (const entry of toRemove) {
|
|
6325
|
+
crontab = removeCronLines(crontab, entry.id, entry.platform);
|
|
6326
|
+
removeRegistered(entry.id);
|
|
6327
|
+
}
|
|
6328
|
+
writeCrontab(crontab);
|
|
6329
|
+
return { removed: toRemove, notFound: false };
|
|
6330
|
+
}
|
|
6331
|
+
function scheduleStatus() {
|
|
6332
|
+
const entries = listSchedules();
|
|
6333
|
+
return entries.map((entry) => {
|
|
6334
|
+
const logExists = fs15.existsSync(entry.logFile);
|
|
6335
|
+
const logSize = logExists ? fs15.statSync(entry.logFile).size : 0;
|
|
6336
|
+
let logTail = [];
|
|
6337
|
+
let lastRunAt = null;
|
|
6338
|
+
if (logExists) {
|
|
6339
|
+
try {
|
|
6340
|
+
lastRunAt = fs15.statSync(entry.logFile).mtime.toISOString();
|
|
6341
|
+
if (logSize > 0) {
|
|
6342
|
+
const content = fs15.readFileSync(entry.logFile, "utf-8");
|
|
6343
|
+
logTail = content.trim().split("\n").slice(-10);
|
|
6344
|
+
}
|
|
6345
|
+
} catch {
|
|
6346
|
+
}
|
|
6347
|
+
}
|
|
6348
|
+
let drift = "ok";
|
|
6349
|
+
if (!entry.cronInstalled) drift = "missing-cron";
|
|
6350
|
+
else if (!fs15.existsSync(entry.nodeBin)) drift = "stale-node-bin";
|
|
6351
|
+
else if (!fs15.existsSync(entry.cliBin)) drift = "stale-cli-bin";
|
|
6352
|
+
return { entry, logExists, logSize, logTail, lastRunAt, drift };
|
|
6353
|
+
});
|
|
6354
|
+
}
|
|
6355
|
+
function repairSchedule(id) {
|
|
6356
|
+
const entry = getRegistered(id);
|
|
6357
|
+
if (!entry) throw new Error(`No registered schedule with id "${id}".`);
|
|
6358
|
+
const healed = {
|
|
6359
|
+
...entry,
|
|
6360
|
+
nodeBin: process.execPath,
|
|
6361
|
+
cliBin: resolveOneBinary()
|
|
6362
|
+
};
|
|
6363
|
+
const crontab = readCrontab();
|
|
6364
|
+
const cleaned = removeCronLines(crontab, healed.id, healed.platform);
|
|
6365
|
+
const next = (cleaned.trimEnd() + "\n" + buildCronLine(healed) + "\n").replace(/^\n+/, "");
|
|
6366
|
+
writeCrontab(next);
|
|
6367
|
+
upsertRegistered(healed);
|
|
6368
|
+
return healed;
|
|
6369
|
+
}
|
|
6370
|
+
|
|
6371
|
+
// src/lib/sync/where-parser.ts
|
|
6372
|
+
function unquote(value) {
|
|
6373
|
+
if (value.length >= 2) {
|
|
6374
|
+
const first = value[0];
|
|
6375
|
+
const last = value[value.length - 1];
|
|
6376
|
+
if (first === "'" && last === "'" || first === '"' && last === '"') {
|
|
6377
|
+
return value.slice(1, -1);
|
|
6378
|
+
}
|
|
6379
|
+
}
|
|
6380
|
+
return value;
|
|
6381
|
+
}
|
|
6382
|
+
function splitConditions(input) {
|
|
6383
|
+
const parts = [];
|
|
6384
|
+
let buf = "";
|
|
6385
|
+
let inSingle = false;
|
|
6386
|
+
let inDouble = false;
|
|
6387
|
+
for (let i = 0; i < input.length; i++) {
|
|
6388
|
+
const ch = input[i];
|
|
6389
|
+
if (ch === "'" && !inDouble) inSingle = !inSingle;
|
|
6390
|
+
else if (ch === '"' && !inSingle) inDouble = !inDouble;
|
|
6391
|
+
if (ch === "," && !inSingle && !inDouble) {
|
|
6392
|
+
if (buf.trim().length > 0) parts.push(buf.trim());
|
|
6393
|
+
buf = "";
|
|
6394
|
+
} else {
|
|
6395
|
+
buf += ch;
|
|
6396
|
+
}
|
|
6397
|
+
}
|
|
6398
|
+
if (buf.trim().length > 0) parts.push(buf.trim());
|
|
6399
|
+
return parts;
|
|
6400
|
+
}
|
|
6401
|
+
function parseCondition(condition) {
|
|
6402
|
+
const operators = [">=", "<=", "!=", ">", "<", "=", " like "];
|
|
6403
|
+
for (const op of operators) {
|
|
6404
|
+
const idx = condition.toLowerCase().indexOf(op.toLowerCase());
|
|
6405
|
+
if (idx > 0) {
|
|
6406
|
+
const field = condition.slice(0, idx).trim();
|
|
6407
|
+
const rawValue = condition.slice(idx + op.length).trim();
|
|
6408
|
+
const value = unquote(rawValue);
|
|
6409
|
+
const sqlOp = op.trim().toUpperCase() === "LIKE" ? "LIKE" : op.trim();
|
|
6410
|
+
return { field, operator: sqlOp, value };
|
|
6411
|
+
}
|
|
6412
|
+
}
|
|
6413
|
+
throw new Error(
|
|
6414
|
+
`Cannot parse where condition: "${condition}". Expected format: field=value, field>value, field like %pattern`
|
|
6415
|
+
);
|
|
6416
|
+
}
|
|
6417
|
+
|
|
6418
|
+
// src/lib/sync/query.ts
|
|
6419
|
+
var COMMON_DATE_COLUMNS = ["created_at", "createdAt", "created", "date", "timestamp", "updated_at", "updatedAt"];
|
|
6420
|
+
function detectDateColumn(columns) {
|
|
6421
|
+
const matches = columns.filter((c) => COMMON_DATE_COLUMNS.includes(c));
|
|
6422
|
+
if (matches.length === 1) return matches[0];
|
|
6423
|
+
if (matches.length > 1) return null;
|
|
6424
|
+
return null;
|
|
6425
|
+
}
|
|
6426
|
+
function formatSyncAge(lastSync) {
|
|
6427
|
+
const diffMs = Date.now() - new Date(lastSync).getTime();
|
|
6428
|
+
const seconds = Math.floor(diffMs / 1e3);
|
|
6429
|
+
if (seconds < 60) return `${seconds}s`;
|
|
6430
|
+
const minutes = Math.floor(seconds / 60);
|
|
6431
|
+
if (minutes < 60) return `${minutes}m`;
|
|
6432
|
+
const hours = Math.floor(minutes / 60);
|
|
6433
|
+
const remainingMinutes = minutes % 60;
|
|
6434
|
+
if (hours < 24) return `${hours}h ${remainingMinutes}m`;
|
|
6435
|
+
const days = Math.floor(hours / 24);
|
|
6436
|
+
return `${days}d ${hours % 24}h`;
|
|
6437
|
+
}
|
|
6438
|
+
async function executeQuery(platform, model, options) {
|
|
6439
|
+
const db = await openDatabase(platform);
|
|
6440
|
+
try {
|
|
6441
|
+
if (!tableExists(db, model)) {
|
|
6442
|
+
throw new Error(`No synced data for ${platform}/${model}. Run 'one sync run ${platform} --models ${model}' first.`);
|
|
6443
|
+
}
|
|
6444
|
+
const columns = getTableColumns(db, model).map((c) => c.name);
|
|
6445
|
+
const whereClauses = [];
|
|
6446
|
+
const params = [];
|
|
6447
|
+
if (options.where) {
|
|
6448
|
+
const conditions = splitConditions(options.where);
|
|
6449
|
+
for (const cond of conditions) {
|
|
6450
|
+
const parsed2 = parseCondition(cond);
|
|
6451
|
+
if (!columns.includes(parsed2.field)) {
|
|
6452
|
+
throw new Error(`Column "${parsed2.field}" not found. Available: ${columns.join(", ")}`);
|
|
6453
|
+
}
|
|
6454
|
+
whereClauses.push(`"${parsed2.field}" ${parsed2.operator} ?`);
|
|
6455
|
+
params.push(parsed2.value);
|
|
6456
|
+
}
|
|
6457
|
+
}
|
|
6458
|
+
if (options.after || options.before) {
|
|
6459
|
+
let dateCol = options.dateField;
|
|
6460
|
+
if (!dateCol) {
|
|
6461
|
+
dateCol = detectDateColumn(columns) ?? void 0;
|
|
6462
|
+
if (!dateCol) {
|
|
6463
|
+
throw new Error(
|
|
6464
|
+
`Cannot auto-detect date column. Use --date-field to specify. Available columns: ${columns.join(", ")}`
|
|
6465
|
+
);
|
|
6466
|
+
}
|
|
6467
|
+
}
|
|
6468
|
+
if (!columns.includes(dateCol)) {
|
|
6469
|
+
throw new Error(`Date column "${dateCol}" not found. Available: ${columns.join(", ")}`);
|
|
6470
|
+
}
|
|
6471
|
+
if (options.after) {
|
|
6472
|
+
whereClauses.push(`"${dateCol}" >= ?`);
|
|
6473
|
+
params.push(options.after);
|
|
6474
|
+
}
|
|
6475
|
+
if (options.before) {
|
|
6476
|
+
whereClauses.push(`"${dateCol}" <= ?`);
|
|
6477
|
+
params.push(options.before);
|
|
6478
|
+
}
|
|
6479
|
+
}
|
|
6480
|
+
const table = sanitizeTableName(model);
|
|
6481
|
+
let sql = `SELECT * FROM "${table}"`;
|
|
6482
|
+
if (whereClauses.length > 0) {
|
|
6483
|
+
sql += ` WHERE ${whereClauses.join(" AND ")}`;
|
|
6484
|
+
}
|
|
6485
|
+
if (options.orderBy) {
|
|
6486
|
+
if (!columns.includes(options.orderBy)) {
|
|
6487
|
+
throw new Error(`Order column "${options.orderBy}" not found. Available: ${columns.join(", ")}`);
|
|
6488
|
+
}
|
|
6489
|
+
const dir = (options.order || "asc").toUpperCase();
|
|
6490
|
+
sql += ` ORDER BY "${options.orderBy}" ${dir}`;
|
|
6491
|
+
}
|
|
6492
|
+
const limit = options.limit ?? 50;
|
|
6493
|
+
sql += ` LIMIT ${limit}`;
|
|
6494
|
+
const results = db.prepare(sql).all(...params);
|
|
6495
|
+
const parsed = results.map((row) => {
|
|
6496
|
+
const out = {};
|
|
6497
|
+
for (const [key, value] of Object.entries(row)) {
|
|
6498
|
+
if (typeof value === "string" && (value.startsWith("{") || value.startsWith("["))) {
|
|
6499
|
+
try {
|
|
6500
|
+
out[key] = JSON.parse(value);
|
|
6501
|
+
} catch {
|
|
6502
|
+
out[key] = value;
|
|
6503
|
+
}
|
|
6504
|
+
} else {
|
|
6505
|
+
out[key] = value;
|
|
6506
|
+
}
|
|
6507
|
+
}
|
|
6508
|
+
return out;
|
|
6509
|
+
});
|
|
6510
|
+
const state = getModelState(platform, model);
|
|
6511
|
+
const lastSync = state?.lastSync ?? null;
|
|
6512
|
+
const syncAge = lastSync ? formatSyncAge(lastSync) : null;
|
|
6513
|
+
db.close();
|
|
6514
|
+
return {
|
|
6515
|
+
platform,
|
|
6516
|
+
model,
|
|
6517
|
+
results: parsed,
|
|
6518
|
+
total: parsed.length,
|
|
6519
|
+
query: sql,
|
|
6520
|
+
source: "local",
|
|
6521
|
+
lastSync,
|
|
6522
|
+
syncAge
|
|
6523
|
+
};
|
|
6524
|
+
} catch (err) {
|
|
6525
|
+
db.close();
|
|
6526
|
+
throw err;
|
|
6527
|
+
}
|
|
6528
|
+
}
|
|
6529
|
+
async function executeRawSql(platform, sql) {
|
|
6530
|
+
const trimmed = sql.trim();
|
|
6531
|
+
const upper = trimmed.toUpperCase();
|
|
6532
|
+
if (!upper.startsWith("SELECT")) {
|
|
6533
|
+
throw new Error("Only SELECT statements are allowed. Sync databases are read-only.");
|
|
6534
|
+
}
|
|
6535
|
+
const forbidden = /\b(PRAGMA|ATTACH|DETACH|INSERT|UPDATE|DELETE|DROP|ALTER|CREATE|REPLACE|VACUUM)\b/;
|
|
6536
|
+
if (forbidden.test(upper)) {
|
|
6537
|
+
throw new Error("Only pure SELECT queries are allowed. PRAGMA/ATTACH/DDL/DML are blocked.");
|
|
6538
|
+
}
|
|
6539
|
+
const db = await openDatabase(platform);
|
|
6540
|
+
try {
|
|
6541
|
+
const results = db.prepare(trimmed).all();
|
|
6542
|
+
const parsed = results.map((row) => {
|
|
6543
|
+
const out = {};
|
|
6544
|
+
for (const [key, value] of Object.entries(row)) {
|
|
6545
|
+
if (typeof value === "string" && (value.startsWith("{") || value.startsWith("["))) {
|
|
6546
|
+
try {
|
|
6547
|
+
out[key] = JSON.parse(value);
|
|
6548
|
+
} catch {
|
|
6549
|
+
out[key] = value;
|
|
6550
|
+
}
|
|
6551
|
+
} else {
|
|
6552
|
+
out[key] = value;
|
|
6553
|
+
}
|
|
6554
|
+
}
|
|
6555
|
+
return out;
|
|
6556
|
+
});
|
|
6557
|
+
db.close();
|
|
6558
|
+
return { results: parsed, query: trimmed };
|
|
6559
|
+
} catch (err) {
|
|
6560
|
+
db.close();
|
|
6561
|
+
throw err;
|
|
6562
|
+
}
|
|
6563
|
+
}
|
|
6564
|
+
|
|
6565
|
+
// src/lib/sync/search.ts
|
|
6566
|
+
async function searchSyncedData(query, options) {
|
|
6567
|
+
const limit = options.limit ?? 20;
|
|
6568
|
+
const platforms = options.platform ? [options.platform] : listSyncedPlatforms();
|
|
6569
|
+
if (platforms.length === 0) {
|
|
6570
|
+
throw new Error("No synced data found. Run 'one sync run <platform>' first.");
|
|
6571
|
+
}
|
|
6572
|
+
const ftsQuery = query.split(/\s+/).map((term) => term.includes("*") || term.includes('"') ? term : `${term}*`).join(" ");
|
|
6573
|
+
const allResults = [];
|
|
6574
|
+
for (const platform of platforms) {
|
|
6575
|
+
const db = await openDatabase(platform);
|
|
6576
|
+
try {
|
|
6577
|
+
const tables = options.models ?? listTables(db);
|
|
6578
|
+
for (const model of tables) {
|
|
6579
|
+
if (!tableExists(db, model)) continue;
|
|
6580
|
+
const ftsTable = `${model}_fts`;
|
|
6581
|
+
const ftsExists = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name=?`).get(ftsTable);
|
|
6582
|
+
if (!ftsExists) continue;
|
|
6583
|
+
try {
|
|
6584
|
+
const rows = db.prepare(`
|
|
6585
|
+
SELECT "${model}".*, "${ftsTable}".rank
|
|
6586
|
+
FROM "${ftsTable}"
|
|
6587
|
+
JOIN "${model}" ON "${model}".rowid = "${ftsTable}".rowid
|
|
6588
|
+
WHERE "${ftsTable}" MATCH ?
|
|
6589
|
+
ORDER BY rank
|
|
6590
|
+
LIMIT ?
|
|
6591
|
+
`).all(ftsQuery, limit);
|
|
6592
|
+
for (const row of rows) {
|
|
6593
|
+
const rank = row.rank;
|
|
6594
|
+
const record = {};
|
|
6595
|
+
for (const [key, value] of Object.entries(row)) {
|
|
6596
|
+
if (key === "rank") continue;
|
|
6597
|
+
if (typeof value === "string" && (value.startsWith("{") || value.startsWith("["))) {
|
|
6598
|
+
try {
|
|
6599
|
+
record[key] = JSON.parse(value);
|
|
6600
|
+
} catch {
|
|
6601
|
+
record[key] = value;
|
|
6602
|
+
}
|
|
6603
|
+
} else {
|
|
6604
|
+
record[key] = value;
|
|
6605
|
+
}
|
|
6606
|
+
}
|
|
6607
|
+
allResults.push({ platform, model, record, rank });
|
|
6608
|
+
}
|
|
6609
|
+
} catch {
|
|
6610
|
+
}
|
|
6611
|
+
}
|
|
6612
|
+
db.close();
|
|
6613
|
+
} catch {
|
|
6614
|
+
try {
|
|
6615
|
+
db.close();
|
|
6616
|
+
} catch {
|
|
6617
|
+
}
|
|
6618
|
+
}
|
|
6619
|
+
}
|
|
6620
|
+
allResults.sort((a, b) => a.rank - b.rank);
|
|
6621
|
+
const limited = allResults.slice(0, limit);
|
|
6622
|
+
return {
|
|
6623
|
+
results: limited,
|
|
6624
|
+
total: limited.length
|
|
6625
|
+
};
|
|
6626
|
+
}
|
|
6627
|
+
|
|
6628
|
+
// src/lib/sync/index.ts
|
|
6629
|
+
import { spawn as spawn4 } from "child_process";
|
|
6630
|
+
import * as p7 from "@clack/prompts";
|
|
6631
|
+
import pc9 from "picocolors";
|
|
6632
|
+
async function syncInstallCommand() {
|
|
6633
|
+
if (await isSqliteAvailable()) {
|
|
6634
|
+
if (isAgentMode()) {
|
|
6635
|
+
json({ status: "already-installed", module: "better-sqlite3" });
|
|
6636
|
+
} else {
|
|
6637
|
+
note2("better-sqlite3 is already installed.", "Sync");
|
|
6638
|
+
}
|
|
6639
|
+
return;
|
|
6640
|
+
}
|
|
6641
|
+
if (!isAgentMode()) {
|
|
6642
|
+
process.stderr.write("Installing better-sqlite3 (native module, may take ~30s)...\n");
|
|
6643
|
+
}
|
|
6644
|
+
const install = (args) => new Promise((resolve) => {
|
|
6645
|
+
const child = spawn4("npm", args, { stdio: isAgentMode() ? "ignore" : "inherit" });
|
|
6646
|
+
child.on("exit", (code2) => resolve(code2 ?? 1));
|
|
6647
|
+
child.on("error", () => resolve(1));
|
|
6648
|
+
});
|
|
6649
|
+
let code = await install(["install", "-g", "better-sqlite3"]);
|
|
6650
|
+
if (code !== 0) {
|
|
6651
|
+
code = await install(["install", "better-sqlite3"]);
|
|
6652
|
+
}
|
|
6653
|
+
if (code !== 0) {
|
|
6654
|
+
error(
|
|
6655
|
+
"Failed to install better-sqlite3. This usually means your system is missing build tools.\nOn macOS: xcode-select --install\nOn Linux: install python3, make, g++\nThen retry: one sync install"
|
|
6656
|
+
);
|
|
6657
|
+
}
|
|
6658
|
+
if (isAgentMode()) {
|
|
6659
|
+
json({ status: "installed", module: "better-sqlite3" });
|
|
6660
|
+
} else {
|
|
6661
|
+
outro2("better-sqlite3 installed. You can now run one sync commands.");
|
|
6662
|
+
}
|
|
6663
|
+
}
|
|
6664
|
+
async function syncDoctorCommand() {
|
|
6665
|
+
const checks = [];
|
|
6666
|
+
try {
|
|
6667
|
+
const Database = await loadSqlite();
|
|
6668
|
+
checks.push({ name: "better-sqlite3 loads", ok: true });
|
|
6669
|
+
try {
|
|
6670
|
+
const db = new Database(":memory:");
|
|
6671
|
+
db.exec("CREATE TABLE t (x INTEGER); INSERT INTO t VALUES (1);");
|
|
6672
|
+
const row = db.prepare("SELECT x FROM t").get();
|
|
6673
|
+
db.close();
|
|
6674
|
+
checks.push({ name: "in-memory DB read/write", ok: row.x === 1 });
|
|
6675
|
+
} catch (err) {
|
|
6676
|
+
checks.push({ name: "in-memory DB read/write", ok: false, detail: err instanceof Error ? err.message : String(err) });
|
|
6677
|
+
}
|
|
6678
|
+
try {
|
|
6679
|
+
const db = new Database(":memory:");
|
|
6680
|
+
db.exec("CREATE VIRTUAL TABLE t USING fts5(content)");
|
|
6681
|
+
db.close();
|
|
6682
|
+
checks.push({ name: "FTS5 virtual table support", ok: true });
|
|
6683
|
+
} catch (err) {
|
|
6684
|
+
checks.push({ name: "FTS5 virtual table support", ok: false, detail: err instanceof Error ? err.message : String(err) });
|
|
6685
|
+
}
|
|
6686
|
+
} catch (err) {
|
|
6687
|
+
checks.push({ name: "better-sqlite3 loads", ok: false, detail: err instanceof Error ? err.message : String(err) });
|
|
6688
|
+
}
|
|
6689
|
+
const allOk = checks.every((c) => c.ok);
|
|
6690
|
+
if (isAgentMode()) {
|
|
6691
|
+
json({ ok: allOk, checks });
|
|
6692
|
+
return;
|
|
6693
|
+
}
|
|
6694
|
+
for (const c of checks) {
|
|
6695
|
+
const mark = c.ok ? pc9.green("\u2713") : pc9.red("\u2717");
|
|
6696
|
+
console.log(` ${mark} ${c.name}${c.detail ? pc9.dim(` \u2014 ${c.detail}`) : ""}`);
|
|
6697
|
+
}
|
|
6698
|
+
if (!allOk) {
|
|
6699
|
+
console.log(`
|
|
6700
|
+
${pc9.yellow("Sync is not ready.")} Try: ${pc9.bold("one sync install")}`);
|
|
6701
|
+
} else {
|
|
6702
|
+
console.log(`
|
|
6703
|
+
${pc9.green("Sync is ready.")}`);
|
|
6704
|
+
}
|
|
6705
|
+
}
|
|
6706
|
+
function getApi() {
|
|
6707
|
+
const apiKey = getApiKey();
|
|
6708
|
+
if (!apiKey) {
|
|
6709
|
+
error('No API key configured. Run "one init" first.');
|
|
6710
|
+
}
|
|
6711
|
+
return new OneApi(apiKey);
|
|
6712
|
+
}
|
|
6713
|
+
async function syncProfilesCommand(platform) {
|
|
6714
|
+
const profiles = listBuiltinProfiles(platform);
|
|
6715
|
+
if (isAgentMode()) {
|
|
6716
|
+
json({
|
|
6717
|
+
profiles: profiles.map((p8) => ({
|
|
6718
|
+
platform: p8.platform,
|
|
6719
|
+
model: p8.model,
|
|
6720
|
+
description: p8.description,
|
|
6721
|
+
hasEnrich: !!p8.enrich,
|
|
6722
|
+
hasIdentityKey: !!p8.identityKey
|
|
6723
|
+
})),
|
|
6724
|
+
total: profiles.length,
|
|
6725
|
+
_hint: profiles.length > 0 ? "Use a built-in profile: one --agent sync init <platform> <model>" : "No built-in profiles found. Use sync init to auto-infer from action knowledge."
|
|
6726
|
+
});
|
|
6727
|
+
return;
|
|
6728
|
+
}
|
|
6729
|
+
if (profiles.length === 0) {
|
|
6730
|
+
note2(
|
|
6731
|
+
platform ? `No built-in profiles for ${platform}. Use sync init to auto-infer from action knowledge.` : "No built-in profiles found.",
|
|
6732
|
+
"Profiles"
|
|
6733
|
+
);
|
|
6734
|
+
return;
|
|
6735
|
+
}
|
|
6736
|
+
for (const p8 of profiles) {
|
|
6737
|
+
const extras = [];
|
|
6738
|
+
if (p8.enrich) extras.push("enrich");
|
|
6739
|
+
if (p8.identityKey) extras.push("identity");
|
|
6740
|
+
if (p8.dateFilter) extras.push("incremental");
|
|
6741
|
+
const tags = extras.length > 0 ? ` ${pc9.dim(`[${extras.join(", ")}]`)}` : "";
|
|
6742
|
+
console.log(` ${pc9.bold(`${p8.platform}/${p8.model}`.padEnd(35))} ${p8.description}${tags}`);
|
|
6743
|
+
}
|
|
6744
|
+
console.log(`
|
|
6745
|
+
${profiles.length} built-in profile(s). Run ${pc9.bold("one sync init <platform> <model>")} to use one.`);
|
|
6746
|
+
}
|
|
6747
|
+
async function syncModelsCommand(platform) {
|
|
6748
|
+
const api = getApi();
|
|
6749
|
+
const spinner5 = createSpinner();
|
|
6750
|
+
spinner5.start(`Discovering models for ${platform}...`);
|
|
6751
|
+
try {
|
|
6752
|
+
const models = await discoverModels(api, platform);
|
|
6753
|
+
spinner5.stop(`Found ${models.length} models`);
|
|
6754
|
+
if (isAgentMode()) {
|
|
6755
|
+
json({ platform, models, total: models.length });
|
|
6756
|
+
return;
|
|
6757
|
+
}
|
|
6758
|
+
if (models.length === 0) {
|
|
6759
|
+
note2("No list-type actions found for this platform.", "Models");
|
|
6760
|
+
return;
|
|
6761
|
+
}
|
|
6762
|
+
const lines = models.map(
|
|
6763
|
+
(m) => ` ${pc9.bold(m.name.padEnd(30))} ${pc9.dim(m.listAction.method)} ${pc9.dim(m.listAction.path)}`
|
|
6764
|
+
);
|
|
6765
|
+
note2(lines.join("\n"), `${platform} \u2014 ${models.length} models`);
|
|
6766
|
+
} catch (err) {
|
|
6767
|
+
spinner5.stop("Failed");
|
|
6768
|
+
error(`Error discovering models: ${err instanceof Error ? err.message : String(err)}`);
|
|
6769
|
+
}
|
|
6770
|
+
}
|
|
6771
|
+
async function syncInitCommand(platform, model, options) {
|
|
6772
|
+
if (!options.config) {
|
|
6773
|
+
const api = getApi();
|
|
6774
|
+
const spinner5 = createSpinner();
|
|
6775
|
+
spinner5.start(`Looking up ${platform}/${model}...`);
|
|
6776
|
+
try {
|
|
6777
|
+
const models = await discoverModels(api, platform);
|
|
6778
|
+
const match = models.find((m) => m.name === model || m.name.toLowerCase() === model.toLowerCase());
|
|
6779
|
+
let actionId;
|
|
6780
|
+
if (match) {
|
|
6781
|
+
actionId = match.listAction.actionId;
|
|
6782
|
+
if (actionId && !actionId.startsWith("conn_mod_def::")) {
|
|
6783
|
+
actionId = void 0;
|
|
6784
|
+
}
|
|
6785
|
+
spinner5.stop(actionId ? "Found model + action ID" : "Found model (action ID not resolved)");
|
|
6786
|
+
} else {
|
|
6787
|
+
spinner5.stop("Model not found in available actions");
|
|
6788
|
+
}
|
|
6789
|
+
const builtin = loadBuiltinProfile(platform, model);
|
|
6790
|
+
let template;
|
|
6791
|
+
let inferred = null;
|
|
6792
|
+
if (builtin) {
|
|
6793
|
+
template = { ...builtin };
|
|
6794
|
+
if (actionId) template.actionId = actionId;
|
|
6795
|
+
delete template.description;
|
|
6796
|
+
inferred = { reasoning: [`Built-in profile found for ${platform}/${model}: "${builtin.description}"`] };
|
|
6797
|
+
} else {
|
|
6798
|
+
template = generateTemplate(platform, model, actionId);
|
|
6799
|
+
}
|
|
6800
|
+
if (actionId && !builtin) {
|
|
6801
|
+
try {
|
|
6802
|
+
const knowledgeResp = await api.getActionKnowledge(actionId);
|
|
6803
|
+
inferred = inferProfileFromKnowledge(knowledgeResp?.knowledge, model, platform);
|
|
6804
|
+
if (inferred.pagination) template.pagination = inferred.pagination;
|
|
6805
|
+
if (inferred.resultsPath) template.resultsPath = inferred.resultsPath;
|
|
6806
|
+
if (inferred.idField) template.idField = inferred.idField;
|
|
6807
|
+
if (inferred.dateFilterParam) {
|
|
6808
|
+
template.dateFilter = { param: inferred.dateFilterParam, format: "iso8601" };
|
|
6809
|
+
}
|
|
6810
|
+
if (inferred.limitLocation) template.limitLocation = inferred.limitLocation;
|
|
6811
|
+
if (inferred.limitParam) template.limitParam = inferred.limitParam;
|
|
6812
|
+
if (inferred.pathVars && Object.keys(inferred.pathVars).length > 0) {
|
|
6813
|
+
template.pathVars = inferred.pathVars;
|
|
6814
|
+
}
|
|
6815
|
+
} catch {
|
|
6816
|
+
}
|
|
6817
|
+
}
|
|
6818
|
+
try {
|
|
6819
|
+
const connections = await api.listConnections();
|
|
6820
|
+
const platformConns = connections.filter(
|
|
6821
|
+
(c) => c.platform === platform
|
|
6822
|
+
);
|
|
6823
|
+
if (platformConns.length === 1) {
|
|
6824
|
+
template.connectionKey = platformConns[0].key;
|
|
6825
|
+
inferred?.reasoning.push(`connectionKey: auto-resolved (only one ${platform} connection)`);
|
|
6826
|
+
} else if (platformConns.length > 1) {
|
|
6827
|
+
inferred?.reasoning.push(
|
|
6828
|
+
`connectionKey: ${platformConns.length} connections found \u2014 pick one from \`one list\``
|
|
6829
|
+
);
|
|
6830
|
+
}
|
|
6831
|
+
} catch {
|
|
6832
|
+
}
|
|
6833
|
+
try {
|
|
6834
|
+
writeDraftProfile(platform, model, template);
|
|
6835
|
+
} catch {
|
|
6836
|
+
}
|
|
6837
|
+
const templateStr = JSON.stringify(template);
|
|
6838
|
+
const isComplete = !templateStr.includes("FILL_IN");
|
|
6839
|
+
let testReport = null;
|
|
6840
|
+
if (isComplete) {
|
|
6841
|
+
try {
|
|
6842
|
+
testReport = await testSyncProfile(api, template);
|
|
6843
|
+
if (testReport.autoFixed && Object.keys(testReport.autoFixed).length > 0) {
|
|
6844
|
+
Object.assign(template, testReport.autoFixed);
|
|
6845
|
+
try {
|
|
6846
|
+
writeDraftProfile(platform, model, template);
|
|
6847
|
+
} catch {
|
|
6848
|
+
}
|
|
6849
|
+
}
|
|
6850
|
+
} catch {
|
|
6851
|
+
}
|
|
6852
|
+
}
|
|
6853
|
+
const hint = isComplete ? testReport?.ok ? `Profile complete and validated. Run: one sync run ${platform} --models ${model}` : `Profile complete but test had issues \u2014 check the test report below.` : actionId ? `Fill remaining FILL_IN fields, then: one sync test ${platform}/${model}` : `Action ID not resolved. Run: one --agent actions search ${platform} "list ${model}" -t execute`;
|
|
6854
|
+
if (isAgentMode()) {
|
|
6855
|
+
json({
|
|
6856
|
+
...template,
|
|
6857
|
+
_hint: hint,
|
|
6858
|
+
_inferred: inferred?.reasoning ?? [],
|
|
6859
|
+
_draft: !isComplete,
|
|
6860
|
+
_complete: isComplete,
|
|
6861
|
+
...testReport ? { _test: { ok: testReport.ok, checks: testReport.checks, autoFixed: testReport.autoFixed } } : {}
|
|
6862
|
+
});
|
|
6863
|
+
} else {
|
|
6864
|
+
note2(JSON.stringify(template, null, 2), "Sync profile template");
|
|
6865
|
+
if (inferred && inferred.reasoning.length > 0) {
|
|
6866
|
+
console.log(`
|
|
6867
|
+
${pc9.bold("Inferred from knowledge:")}`);
|
|
6868
|
+
for (const r of inferred.reasoning) console.log(` ${pc9.dim("\u2022")} ${r}`);
|
|
6869
|
+
}
|
|
6870
|
+
if (testReport) {
|
|
6871
|
+
console.log(`
|
|
6872
|
+
${pc9.bold("Test results:")}`);
|
|
6873
|
+
for (const c of testReport.checks) {
|
|
6874
|
+
const mark = c.ok ? pc9.green("\u2713") : pc9.red("\u2717");
|
|
6875
|
+
console.log(` ${mark} ${c.name}${c.detail ? pc9.dim(` \u2014 ${c.detail}`) : ""}`);
|
|
6876
|
+
}
|
|
6877
|
+
}
|
|
6878
|
+
console.log(`
|
|
6879
|
+
${hint}`);
|
|
6880
|
+
if (!isComplete) {
|
|
6881
|
+
console.log(`
|
|
6882
|
+
Run with --config to save:
|
|
6883
|
+
one sync init ${platform} ${model} --config '${JSON.stringify(template)}'`);
|
|
6884
|
+
}
|
|
6885
|
+
}
|
|
6886
|
+
} catch (err) {
|
|
6887
|
+
spinner5.stop("Failed");
|
|
6888
|
+
error(`Error: ${err instanceof Error ? err.message : String(err)}`);
|
|
6889
|
+
}
|
|
6890
|
+
return;
|
|
6891
|
+
}
|
|
6892
|
+
let patch;
|
|
6893
|
+
try {
|
|
6894
|
+
patch = JSON.parse(options.config);
|
|
6895
|
+
} catch {
|
|
6896
|
+
error("Invalid JSON in --config. Provide a valid JSON sync profile.");
|
|
6897
|
+
return;
|
|
6898
|
+
}
|
|
6899
|
+
const existing = readProfile(platform, model);
|
|
6900
|
+
const profile = {
|
|
6901
|
+
...existing ?? {},
|
|
6902
|
+
...patch,
|
|
6903
|
+
// Ensure platform/model from args always win
|
|
6904
|
+
platform,
|
|
6905
|
+
model,
|
|
6906
|
+
// Deep-merge pagination so you can patch just nextPath without losing type
|
|
6907
|
+
pagination: {
|
|
6908
|
+
...existing?.pagination ?? {},
|
|
6909
|
+
...patch.pagination ?? {}
|
|
6910
|
+
}
|
|
6911
|
+
};
|
|
6912
|
+
try {
|
|
6913
|
+
writeProfile(profile);
|
|
6914
|
+
if (isAgentMode()) {
|
|
6915
|
+
json({ status: "created", platform, model, profile });
|
|
6916
|
+
} else {
|
|
6917
|
+
outro2(`Sync profile saved for ${platform}/${model}`);
|
|
6918
|
+
}
|
|
6919
|
+
} catch (err) {
|
|
6920
|
+
error(`Error saving profile: ${err instanceof Error ? err.message : String(err)}`);
|
|
6921
|
+
}
|
|
6922
|
+
}
|
|
6923
|
+
async function syncTestCommand(platformModel) {
|
|
6924
|
+
const [platform, model] = platformModel.split("/");
|
|
6925
|
+
if (!platform || !model) {
|
|
6926
|
+
error("Usage: one sync test <platform>/<model>. Example: one sync test shopify/orders");
|
|
6927
|
+
}
|
|
6928
|
+
const profile = readProfile(platform, model);
|
|
6929
|
+
if (!profile) {
|
|
6930
|
+
error(
|
|
6931
|
+
`No sync profile found for ${platform}/${model}. Create one with: one sync init ${platform} ${model} --config '...'`
|
|
6932
|
+
);
|
|
6933
|
+
}
|
|
6934
|
+
const api = getApi();
|
|
6935
|
+
const report = await testSyncProfile(api, profile);
|
|
6936
|
+
if (report.autoFixed && Object.keys(report.autoFixed).length > 0) {
|
|
6937
|
+
try {
|
|
6938
|
+
const existing = readProfile(platform, model);
|
|
6939
|
+
if (existing) {
|
|
6940
|
+
const patched = { ...existing, ...report.autoFixed };
|
|
6941
|
+
writeProfile(patched);
|
|
6942
|
+
}
|
|
6943
|
+
} catch {
|
|
6944
|
+
}
|
|
6945
|
+
}
|
|
6946
|
+
if (isAgentMode()) {
|
|
6947
|
+
json(report);
|
|
6948
|
+
return;
|
|
6949
|
+
}
|
|
6950
|
+
for (const c of report.checks) {
|
|
6951
|
+
const mark = c.ok ? pc9.green("\u2713") : pc9.red("\u2717");
|
|
6952
|
+
console.log(` ${mark} ${c.name}${c.detail ? pc9.dim(` \u2014 ${c.detail}`) : ""}`);
|
|
6953
|
+
}
|
|
6954
|
+
if (report.detectedColumns && report.detectedColumns.length > 0) {
|
|
6955
|
+
console.log(`
|
|
6956
|
+
${pc9.bold("Detected columns:")}`);
|
|
6957
|
+
for (const col of report.detectedColumns.slice(0, 20)) {
|
|
6958
|
+
console.log(` ${col.name.padEnd(30)} ${pc9.dim(col.type)}`);
|
|
6959
|
+
}
|
|
6960
|
+
if (report.detectedColumns.length > 20) {
|
|
6961
|
+
console.log(pc9.dim(` ... and ${report.detectedColumns.length - 20} more`));
|
|
6962
|
+
}
|
|
6963
|
+
}
|
|
6964
|
+
console.log(
|
|
6965
|
+
`
|
|
6966
|
+
${report.ok ? pc9.green("Profile looks good.") : pc9.red("Profile has issues.")} ` + (report.ok ? `Run: ${pc9.bold(`one sync run ${platform} --models ${model}`)}` : "Fix the issues above and test again.")
|
|
6967
|
+
);
|
|
6968
|
+
}
|
|
6969
|
+
async function syncRunCommand(platform, options) {
|
|
6970
|
+
const api = getApi();
|
|
6971
|
+
const profiles = listProfiles(platform);
|
|
6972
|
+
const targetModels = options.models;
|
|
6973
|
+
const toSync = targetModels ? profiles.filter((p8) => targetModels.includes(p8.model)) : profiles;
|
|
6974
|
+
if (toSync.length === 0) {
|
|
6975
|
+
error(
|
|
6976
|
+
`No sync profiles found for ${platform}` + (targetModels ? ` with models: ${targetModels.join(", ")}` : "") + `. Run 'one sync init ${platform} <model> --config ...' first.`
|
|
6977
|
+
);
|
|
6978
|
+
}
|
|
6979
|
+
const results = [];
|
|
6980
|
+
for (const profile of toSync) {
|
|
6981
|
+
try {
|
|
6982
|
+
const result = await syncModel(api, profile, options);
|
|
6983
|
+
results.push(result);
|
|
6984
|
+
} catch (err) {
|
|
6985
|
+
const errObj = err;
|
|
6986
|
+
results.push({
|
|
6987
|
+
model: profile.model,
|
|
6988
|
+
recordsSynced: errObj?._recordsSynced ?? 0,
|
|
6989
|
+
pagesProcessed: errObj?._pagesProcessed ?? 0,
|
|
6990
|
+
duration: "0s",
|
|
6991
|
+
status: "failed",
|
|
6992
|
+
error: err instanceof Error ? err.message : String(err)
|
|
6993
|
+
});
|
|
6994
|
+
}
|
|
6995
|
+
}
|
|
6996
|
+
if (isAgentMode()) {
|
|
6997
|
+
json({ platform, results });
|
|
6998
|
+
return;
|
|
6999
|
+
}
|
|
7000
|
+
for (const r of results) {
|
|
7001
|
+
const status = r.status === "complete" ? pc9.green("complete") : r.status === "dry-run" ? pc9.yellow("dry-run") : pc9.red("failed");
|
|
7002
|
+
console.log(` ${pc9.bold(r.model)} \u2014 ${r.recordsSynced} records, ${r.pagesProcessed} pages, ${r.duration} [${status}]`);
|
|
7003
|
+
if ("error" in r && r.error) {
|
|
7004
|
+
console.log(` ${pc9.red(r.error)}`);
|
|
7005
|
+
}
|
|
7006
|
+
}
|
|
7007
|
+
}
|
|
7008
|
+
async function syncQueryCommand(platformModel, options) {
|
|
7009
|
+
const [platform, model] = platformModel.split("/");
|
|
7010
|
+
if (!platform || !model) {
|
|
7011
|
+
error("Usage: one sync query <platform>/<model>. Example: one sync query shopify/orders");
|
|
7012
|
+
}
|
|
7013
|
+
if (options.refresh) {
|
|
7014
|
+
const api = getApi();
|
|
7015
|
+
const profile = readProfile(platform, model);
|
|
7016
|
+
if (profile) {
|
|
7017
|
+
if (!isAgentMode()) {
|
|
7018
|
+
process.stderr.write(`Refreshing ${platform}/${model}...
|
|
7019
|
+
`);
|
|
7020
|
+
}
|
|
7021
|
+
await syncModel(api, profile, { force: options.refreshForce });
|
|
7022
|
+
}
|
|
7023
|
+
}
|
|
7024
|
+
try {
|
|
7025
|
+
const result = await executeQuery(platform, model, options);
|
|
7026
|
+
if (isAgentMode()) {
|
|
7027
|
+
json(result);
|
|
7028
|
+
return;
|
|
7029
|
+
}
|
|
7030
|
+
console.log(pc9.dim(`Query: ${result.query}`));
|
|
7031
|
+
console.log(pc9.dim(`Source: local | Last sync: ${result.lastSync ?? "never"} | Age: ${result.syncAge ?? "n/a"}`));
|
|
7032
|
+
console.log(JSON.stringify(result.results, null, 2));
|
|
7033
|
+
console.log(`
|
|
7034
|
+
${result.total} results`);
|
|
7035
|
+
} catch (err) {
|
|
7036
|
+
error(`Query error: ${err instanceof Error ? err.message : String(err)}`);
|
|
7037
|
+
}
|
|
7038
|
+
}
|
|
7039
|
+
async function syncSearchCommand(query, options) {
|
|
7040
|
+
const modelList = options.models?.split(",").map((m) => m.trim());
|
|
7041
|
+
const limit = options.limit ? parseInt(options.limit, 10) : 20;
|
|
7042
|
+
try {
|
|
7043
|
+
const result = await searchSyncedData(query, { platform: options.platform, models: modelList, limit });
|
|
7044
|
+
if (isAgentMode()) {
|
|
7045
|
+
json(result);
|
|
7046
|
+
return;
|
|
7047
|
+
}
|
|
7048
|
+
if (result.results.length === 0) {
|
|
7049
|
+
note2("No results found.", "Search");
|
|
7050
|
+
return;
|
|
7051
|
+
}
|
|
7052
|
+
for (const r of result.results) {
|
|
7053
|
+
console.log(` ${pc9.bold(`${r.platform}/${r.model}`)} ${pc9.dim(`(rank: ${r.rank.toFixed(2)})`)}`);
|
|
7054
|
+
console.log(` ${JSON.stringify(r.record)}`);
|
|
7055
|
+
}
|
|
7056
|
+
console.log(`
|
|
7057
|
+
${result.total} results`);
|
|
7058
|
+
} catch (err) {
|
|
7059
|
+
error(`Search error: ${err instanceof Error ? err.message : String(err)}`);
|
|
7060
|
+
}
|
|
7061
|
+
}
|
|
7062
|
+
async function syncSqlCommand(platform, sql) {
|
|
7063
|
+
try {
|
|
7064
|
+
const result = await executeRawSql(platform, sql);
|
|
7065
|
+
if (isAgentMode()) {
|
|
7066
|
+
json({ platform, ...result, total: result.results.length });
|
|
7067
|
+
return;
|
|
7068
|
+
}
|
|
7069
|
+
console.log(JSON.stringify(result.results, null, 2));
|
|
7070
|
+
console.log(`
|
|
7071
|
+
${result.results.length} rows`);
|
|
7072
|
+
} catch (err) {
|
|
7073
|
+
error(`SQL error: ${err instanceof Error ? err.message : String(err)}`);
|
|
7074
|
+
}
|
|
7075
|
+
}
|
|
7076
|
+
async function syncDeleteCommand(platformModel, options) {
|
|
7077
|
+
const [platform, model] = platformModel.split("/");
|
|
7078
|
+
if (!platform || !model) {
|
|
7079
|
+
error('Usage: one sync delete <platform>/<model> --id <id> or --where "field=value"');
|
|
7080
|
+
}
|
|
7081
|
+
if (!options.id && !options.where && !options.whereSql) {
|
|
7082
|
+
error('Provide --id <value>, --where "field=value", or --where-sql "SQL predicate" to specify which records to delete.');
|
|
7083
|
+
}
|
|
7084
|
+
const db = await openDatabase(platform);
|
|
7085
|
+
try {
|
|
7086
|
+
if (!tableExists(db, model)) {
|
|
7087
|
+
db.close();
|
|
7088
|
+
error(`No synced data for ${platform}/${model}.`);
|
|
7089
|
+
}
|
|
7090
|
+
let where;
|
|
7091
|
+
let params;
|
|
7092
|
+
if (options.id) {
|
|
7093
|
+
const profile = readProfile(platform, model);
|
|
7094
|
+
const idField = profile?.idField || "id";
|
|
7095
|
+
where = `"${idField}" = ?`;
|
|
7096
|
+
params = [options.id];
|
|
7097
|
+
} else if (options.whereSql) {
|
|
7098
|
+
where = options.whereSql;
|
|
7099
|
+
params = [];
|
|
7100
|
+
} else {
|
|
7101
|
+
const conditions = splitConditions(options.where);
|
|
7102
|
+
const clauses = [];
|
|
7103
|
+
params = [];
|
|
7104
|
+
for (const cond of conditions) {
|
|
7105
|
+
try {
|
|
7106
|
+
const parsed = parseCondition(cond);
|
|
7107
|
+
clauses.push(`"${parsed.field}" ${parsed.operator} ?`);
|
|
7108
|
+
params.push(parsed.value);
|
|
7109
|
+
} catch (err) {
|
|
7110
|
+
db.close();
|
|
7111
|
+
error(err instanceof Error ? err.message : String(err));
|
|
7112
|
+
}
|
|
7113
|
+
}
|
|
7114
|
+
where = clauses.join(" AND ");
|
|
7115
|
+
}
|
|
7116
|
+
const safeTable = sanitizeTableName(model);
|
|
7117
|
+
const preview = db.prepare(`SELECT COUNT(*) as count FROM "${safeTable}" WHERE ${where}`).get(...params);
|
|
7118
|
+
if (preview.count === 0) {
|
|
7119
|
+
db.close();
|
|
7120
|
+
if (isAgentMode()) {
|
|
7121
|
+
json({ deleted: 0, platform, model });
|
|
7122
|
+
} else {
|
|
7123
|
+
console.log("No matching records found.");
|
|
7124
|
+
}
|
|
7125
|
+
return;
|
|
7126
|
+
}
|
|
7127
|
+
if (!options.yes && !isAgentMode()) {
|
|
7128
|
+
const confirmed = await p7.confirm({ message: `Delete ${preview.count} record(s) from ${platform}/${model}?` });
|
|
7129
|
+
if (p7.isCancel(confirmed) || !confirmed) {
|
|
7130
|
+
db.close();
|
|
7131
|
+
cancel2("Cancelled.");
|
|
7132
|
+
return;
|
|
7133
|
+
}
|
|
7134
|
+
}
|
|
7135
|
+
const deleted = deleteRecords(db, model, where, params);
|
|
7136
|
+
rebuildFtsIndex(db, model);
|
|
7137
|
+
db.close();
|
|
7138
|
+
if (isAgentMode()) {
|
|
7139
|
+
json({ deleted, platform, model });
|
|
7140
|
+
} else {
|
|
7141
|
+
console.log(`Deleted ${deleted} record(s) from ${platform}/${model}.`);
|
|
7142
|
+
}
|
|
7143
|
+
} catch (err) {
|
|
7144
|
+
db.close();
|
|
7145
|
+
error(`Delete error: ${err instanceof Error ? err.message : String(err)}`);
|
|
7146
|
+
}
|
|
7147
|
+
}
|
|
7148
|
+
async function syncListCommand(platform) {
|
|
7149
|
+
const profiles = listProfiles(platform);
|
|
7150
|
+
const state = readSyncState();
|
|
7151
|
+
const syncs = profiles.map((p8) => {
|
|
7152
|
+
const modelState = state[p8.platform]?.[p8.model];
|
|
7153
|
+
return {
|
|
7154
|
+
platform: p8.platform,
|
|
7155
|
+
model: p8.model,
|
|
7156
|
+
lastSync: modelState?.lastSync ?? null,
|
|
7157
|
+
totalRecords: modelState?.totalRecords ?? 0,
|
|
7158
|
+
pagesProcessed: modelState?.pagesProcessed ?? 0,
|
|
7159
|
+
dbSize: getDatabaseSize(p8.platform),
|
|
7160
|
+
status: modelState?.status ?? "idle"
|
|
7161
|
+
};
|
|
7162
|
+
});
|
|
7163
|
+
if (isAgentMode()) {
|
|
7164
|
+
json({ syncs });
|
|
7165
|
+
return;
|
|
7166
|
+
}
|
|
7167
|
+
if (syncs.length === 0) {
|
|
7168
|
+
note2("No sync profiles configured.", "Sync");
|
|
7169
|
+
return;
|
|
7170
|
+
}
|
|
7171
|
+
for (const s of syncs) {
|
|
7172
|
+
const status = s.status === "idle" ? pc9.green("idle") : s.status === "syncing" ? pc9.yellow(`syncing \u2014 page ${s.pagesProcessed}`) : pc9.red("failed");
|
|
7173
|
+
console.log(
|
|
7174
|
+
` ${pc9.bold(`${s.platform}/${s.model}`.padEnd(35))} ${String(s.totalRecords).padStart(8)} records ${s.dbSize.padStart(10)} ${status} ${pc9.dim(s.lastSync ? `last: ${s.lastSync}` : "never synced")}`
|
|
7175
|
+
);
|
|
7176
|
+
}
|
|
7177
|
+
}
|
|
7178
|
+
async function syncRemoveCommand(platform, options) {
|
|
7179
|
+
const modelList = options.models?.split(",").map((m) => m.trim());
|
|
7180
|
+
const preview = [];
|
|
7181
|
+
const profiles = listProfiles(platform);
|
|
7182
|
+
const targetModels = modelList ?? profiles.map((p8) => p8.model);
|
|
7183
|
+
try {
|
|
7184
|
+
if (targetModels.length > 0) {
|
|
7185
|
+
const db = await openDatabase(platform);
|
|
7186
|
+
for (const model of targetModels) {
|
|
7187
|
+
preview.push({ model, records: tableExists(db, model) ? countRecords(db, model) : 0 });
|
|
7188
|
+
}
|
|
7189
|
+
db.close();
|
|
7190
|
+
}
|
|
7191
|
+
} catch {
|
|
7192
|
+
for (const model of targetModels) preview.push({ model, records: 0 });
|
|
7193
|
+
}
|
|
7194
|
+
const totalRecords = preview.reduce((sum, p8) => sum + p8.records, 0);
|
|
7195
|
+
const dbSize = getDatabaseSize(platform);
|
|
7196
|
+
if (options.dryRun) {
|
|
7197
|
+
if (isAgentMode()) {
|
|
7198
|
+
json({ dryRun: true, platform, models: preview, totalRecords, dbSize });
|
|
7199
|
+
} else {
|
|
7200
|
+
note2(
|
|
7201
|
+
preview.map((p8) => ` ${p8.model.padEnd(30)} ${String(p8.records).padStart(8)} records`).join("\n") + `
|
|
7202
|
+
|
|
7203
|
+
Total: ${totalRecords} records across ${preview.length} model(s), ${dbSize} on disk`,
|
|
7204
|
+
`Would remove from ${platform}`
|
|
7205
|
+
);
|
|
7206
|
+
}
|
|
7207
|
+
return;
|
|
7208
|
+
}
|
|
7209
|
+
if (!options.yes && !isAgentMode()) {
|
|
7210
|
+
const target = modelList ? `${platform}/${modelList.join(", ")}` : `all synced data for ${platform}`;
|
|
7211
|
+
const confirmed = await p7.confirm({
|
|
7212
|
+
message: `Remove ${target}? (${totalRecords} records, ${dbSize} on disk)`
|
|
7213
|
+
});
|
|
7214
|
+
if (p7.isCancel(confirmed) || !confirmed) {
|
|
7215
|
+
cancel2("Cancelled.");
|
|
7216
|
+
return;
|
|
7217
|
+
}
|
|
7218
|
+
}
|
|
7219
|
+
try {
|
|
7220
|
+
if (modelList) {
|
|
7221
|
+
const db = await openDatabase(platform);
|
|
7222
|
+
for (const model of modelList) {
|
|
7223
|
+
dropTable(db, model);
|
|
7224
|
+
removeModelState(platform, model);
|
|
7225
|
+
}
|
|
7226
|
+
db.close();
|
|
7227
|
+
} else {
|
|
7228
|
+
deleteDatabase(platform);
|
|
7229
|
+
removeModelState(platform);
|
|
7230
|
+
}
|
|
7231
|
+
if (isAgentMode()) {
|
|
7232
|
+
json({ status: "removed", platform, models: modelList ?? "all" });
|
|
7233
|
+
} else {
|
|
7234
|
+
outro2(`Removed sync data for ${platform}${modelList ? `/${modelList.join(", ")}` : ""}`);
|
|
7235
|
+
}
|
|
7236
|
+
} catch (err) {
|
|
7237
|
+
error(`Error: ${err instanceof Error ? err.message : String(err)}`);
|
|
7238
|
+
}
|
|
7239
|
+
}
|
|
7240
|
+
async function syncScheduleAddCommand(platform, options) {
|
|
7241
|
+
try {
|
|
7242
|
+
const { entry, replaced } = addSchedule({
|
|
7243
|
+
platform,
|
|
7244
|
+
every: options.every,
|
|
7245
|
+
models: options.models?.split(",").map((m) => m.trim())
|
|
7246
|
+
});
|
|
7247
|
+
if (isAgentMode()) {
|
|
7248
|
+
json({ status: replaced ? "replaced" : "scheduled", ...entry });
|
|
7249
|
+
return;
|
|
7250
|
+
}
|
|
7251
|
+
const verb = replaced ? "Replaced existing schedule" : "Scheduled sync";
|
|
7252
|
+
outro2(
|
|
7253
|
+
`${verb} ${pc9.bold(entry.id)} \u2014 every ${pc9.bold(entry.every)} (cron: ${pc9.dim(entry.cronExpr)})
|
|
7254
|
+
Logs: ${pc9.dim(entry.logFile)}`
|
|
7255
|
+
);
|
|
7256
|
+
} catch (err) {
|
|
7257
|
+
error(err instanceof Error ? err.message : String(err));
|
|
7258
|
+
}
|
|
7259
|
+
}
|
|
7260
|
+
async function syncScheduleListCommand() {
|
|
7261
|
+
try {
|
|
7262
|
+
const entries = listSchedules();
|
|
7263
|
+
if (isAgentMode()) {
|
|
7264
|
+
json({ schedules: entries });
|
|
7265
|
+
return;
|
|
7266
|
+
}
|
|
7267
|
+
if (entries.length === 0) {
|
|
7268
|
+
note2("No scheduled syncs. Add one with: one sync schedule add <platform> --every 1h", "Schedule");
|
|
7269
|
+
return;
|
|
7270
|
+
}
|
|
7271
|
+
for (const e of entries) {
|
|
7272
|
+
const modelsStr = e.models ? ` [${e.models.join(",")}]` : "";
|
|
7273
|
+
const installed = e.cronInstalled ? pc9.green("\u25CF") : pc9.red("\u2717");
|
|
7274
|
+
console.log(
|
|
7275
|
+
` ${installed} ${pc9.bold(e.id.padEnd(32))} every ${pc9.bold(e.every.padEnd(5))} ${pc9.dim(e.cronExpr.padEnd(13))}${modelsStr}`
|
|
7276
|
+
);
|
|
7277
|
+
console.log(` ${pc9.dim("cwd:")} ${e.cwd}`);
|
|
7278
|
+
}
|
|
7279
|
+
console.log(`
|
|
7280
|
+
${pc9.dim("\u25CF = cron line installed \u2717 = registry drift, run `sync schedule repair <id>`")}`);
|
|
7281
|
+
} catch (err) {
|
|
7282
|
+
error(err instanceof Error ? err.message : String(err));
|
|
7283
|
+
}
|
|
7284
|
+
}
|
|
7285
|
+
async function syncScheduleRemoveCommand(idOrPlatform, options) {
|
|
7286
|
+
try {
|
|
7287
|
+
const result = removeSchedule(idOrPlatform, { allProjects: options.all });
|
|
7288
|
+
if (isAgentMode()) {
|
|
7289
|
+
json({
|
|
7290
|
+
status: result.notFound ? "not-found" : "removed",
|
|
7291
|
+
idOrPlatform,
|
|
7292
|
+
removed: result.removed.map((r) => ({ id: r.id, platform: r.platform, cwd: r.cwd }))
|
|
7293
|
+
});
|
|
7294
|
+
return;
|
|
7295
|
+
}
|
|
7296
|
+
if (result.notFound) {
|
|
7297
|
+
note2(
|
|
7298
|
+
`No scheduled sync found for "${idOrPlatform}"${options.all ? "" : " in this directory"}. Run \`one sync schedule list\` to see all schedules, or pass --all to match across projects.`,
|
|
7299
|
+
"Schedule"
|
|
7300
|
+
);
|
|
7301
|
+
return;
|
|
7302
|
+
}
|
|
7303
|
+
for (const r of result.removed) {
|
|
7304
|
+
console.log(` ${pc9.green("\u2713")} removed ${pc9.bold(r.id)} ${pc9.dim(`(${r.cwd})`)}`);
|
|
7305
|
+
}
|
|
7306
|
+
} catch (err) {
|
|
7307
|
+
error(err instanceof Error ? err.message : String(err));
|
|
7308
|
+
}
|
|
7309
|
+
}
|
|
7310
|
+
async function syncScheduleStatusCommand() {
|
|
7311
|
+
try {
|
|
7312
|
+
const statuses = scheduleStatus();
|
|
7313
|
+
if (isAgentMode()) {
|
|
7314
|
+
json({ schedules: statuses });
|
|
7315
|
+
return;
|
|
7316
|
+
}
|
|
7317
|
+
if (statuses.length === 0) {
|
|
7318
|
+
note2("No scheduled syncs.", "Schedule");
|
|
7319
|
+
return;
|
|
7320
|
+
}
|
|
7321
|
+
for (const s of statuses) {
|
|
7322
|
+
const driftMarker = s.drift === "ok" ? pc9.green("\u25CF") : s.drift === "missing-cron" ? pc9.red("\u2717 missing cron line") : pc9.yellow(`\u26A0 ${s.drift}`);
|
|
7323
|
+
console.log(` ${driftMarker} ${pc9.bold(s.entry.id)} \u2014 every ${s.entry.every} (${pc9.dim(s.entry.cronExpr)})`);
|
|
7324
|
+
console.log(` ${pc9.dim("cwd:")} ${s.entry.cwd}`);
|
|
7325
|
+
console.log(` ${pc9.dim("last run:")} ${s.lastRunAt ?? pc9.yellow("never")}`);
|
|
7326
|
+
console.log(` ${pc9.dim("log:")} ${s.entry.logFile} ${s.logExists ? pc9.dim(`(${s.logSize} bytes)`) : pc9.yellow("(empty)")}`);
|
|
7327
|
+
if (s.logTail.length > 0) {
|
|
7328
|
+
console.log(pc9.dim(" last lines:"));
|
|
7329
|
+
for (const line of s.logTail.slice(-3)) {
|
|
7330
|
+
console.log(pc9.dim(` ${line}`));
|
|
7331
|
+
}
|
|
7332
|
+
}
|
|
7333
|
+
}
|
|
7334
|
+
if (statuses.some((s) => s.drift !== "ok")) {
|
|
7335
|
+
console.log(`
|
|
7336
|
+
${pc9.yellow("Drift detected.")} Run ${pc9.bold("one sync schedule repair <id>")} to heal.`);
|
|
7337
|
+
}
|
|
7338
|
+
} catch (err) {
|
|
7339
|
+
error(err instanceof Error ? err.message : String(err));
|
|
7340
|
+
}
|
|
7341
|
+
}
|
|
7342
|
+
async function syncScheduleRepairCommand(id) {
|
|
7343
|
+
try {
|
|
7344
|
+
const healed = repairSchedule(id);
|
|
7345
|
+
if (isAgentMode()) {
|
|
7346
|
+
json({ status: "repaired", ...healed });
|
|
7347
|
+
return;
|
|
7348
|
+
}
|
|
7349
|
+
outro2(`Repaired ${pc9.bold(healed.id)}: re-installed cron line with current node/cli paths.`);
|
|
7350
|
+
} catch (err) {
|
|
7351
|
+
error(err instanceof Error ? err.message : String(err));
|
|
7352
|
+
}
|
|
7353
|
+
}
|
|
7354
|
+
function registerSyncCommands(program2) {
|
|
7355
|
+
const sync = program2.command("sync").alias("s").description("Sync platform data locally for instant offline queries");
|
|
7356
|
+
sync.command("install").description("Install the local sync engine (better-sqlite3 native module)").action(async () => {
|
|
7357
|
+
await syncInstallCommand();
|
|
7358
|
+
});
|
|
7359
|
+
sync.command("doctor").description("Verify the local sync engine is installed and working").action(async () => {
|
|
7360
|
+
await syncDoctorCommand();
|
|
7361
|
+
});
|
|
7362
|
+
sync.command("profiles [platform]").description("List built-in sync profiles (pre-validated configs for common platforms)").action(async (platform) => {
|
|
7363
|
+
await syncProfilesCommand(platform);
|
|
7364
|
+
});
|
|
7365
|
+
sync.command("models <platform>").description("Discover available data models for a platform").action(async (platform) => {
|
|
7366
|
+
await syncModelsCommand(platform);
|
|
7367
|
+
});
|
|
7368
|
+
sync.command("init <platform> <model>").description("Create or update a sync profile for a model").option("--config <json>", "Sync profile configuration as JSON").action(async (platform, model, options) => {
|
|
7369
|
+
await syncInitCommand(platform, model, options);
|
|
7370
|
+
});
|
|
7371
|
+
sync.command("test <platform/model>").description("Validate a sync profile with a single-page fetch (no DB writes)").action(async (platformModel) => {
|
|
7372
|
+
await syncTestCommand(platformModel);
|
|
7373
|
+
});
|
|
7374
|
+
sync.command("run <platform>").description("Run sync for a platform (syncs all configured models, or specify --models)").option("--models <models>", "Comma-separated list of models to sync").option("--since <duration>", "Sync records since duration (e.g. 90d, 30d, 7d) or date").option("--force", "Ignore existing sync state and start fresh").option("--max-pages <n>", "Maximum number of pages to fetch").option("--dry-run", "Fetch first page only, show results without persisting").option("--full-refresh", "Fetch ALL records and delete local rows no longer in the source (handles deletions)").action(async (platform, options) => {
|
|
7375
|
+
await syncRunCommand(platform, {
|
|
7376
|
+
models: options.models?.split(",").map((m) => m.trim()),
|
|
7377
|
+
since: options.since,
|
|
7378
|
+
force: options.force,
|
|
7379
|
+
maxPages: options.maxPages ? parseInt(options.maxPages, 10) : void 0,
|
|
7380
|
+
dryRun: options.dryRun,
|
|
7381
|
+
fullRefresh: options.fullRefresh
|
|
7382
|
+
});
|
|
7383
|
+
});
|
|
7384
|
+
sync.command("query <platform/model>").description('Query local synced data (e.g. one sync query shopify/orders --where "status=unfulfilled")').option("--where <conditions>", 'Filter conditions (e.g. "status=active,plan=pro")').option("--after <date>", "Records after this date").option("--before <date>", "Records before this date").option("--limit <n>", "Max results (default: 50)").option("--order-by <field>", "Sort by field").option("--order <dir>", "Sort direction: asc or desc").option("--refresh", "Trigger incremental sync before querying").option("--refresh-force", "Trigger full re-sync before querying (implies --refresh)").option("--date-field <field>", "Specify date column for --after/--before").action(async (platformModel, options) => {
|
|
7385
|
+
await syncQueryCommand(platformModel, {
|
|
7386
|
+
where: options.where,
|
|
7387
|
+
after: options.after,
|
|
7388
|
+
before: options.before,
|
|
7389
|
+
limit: options.limit ? parseInt(options.limit, 10) : void 0,
|
|
7390
|
+
orderBy: options.orderBy,
|
|
7391
|
+
order: options.order,
|
|
7392
|
+
refresh: options.refresh || options.refreshForce,
|
|
7393
|
+
refreshForce: options.refreshForce,
|
|
7394
|
+
dateField: options.dateField
|
|
7395
|
+
});
|
|
7396
|
+
});
|
|
7397
|
+
sync.command("search <query>").description("Full-text search across all synced data (or filter by --platform / --models)").option("--platform <platform>", "Search only this platform (default: all)").option("--models <models>", "Comma-separated list of models to search").option("--limit <n>", "Max results (default: 20)").action(async (query, options) => {
|
|
7398
|
+
await syncSearchCommand(query, options);
|
|
7399
|
+
});
|
|
7400
|
+
sync.command("sql <platform> <sql>").description("Execute raw SQL against local sync database (SELECT only)").action(async (platform, sql) => {
|
|
7401
|
+
await syncSqlCommand(platform, sql);
|
|
7402
|
+
});
|
|
7403
|
+
sync.command("delete <platform/model>").description('Delete records from local sync data (e.g. one sync delete notion/pages --id "abc-123")').option("--id <value>", "Delete record by ID").option("--where <conditions>", 'Delete records matching conditions (e.g. "status=archived")').option("--where-sql <predicate>", `Delete using a raw SQL WHERE clause (e.g. "json_extract(data, '$.type') = 'promotion'")`).option("--yes", "Skip confirmation prompt").action(async (platformModel, options) => {
|
|
7404
|
+
await syncDeleteCommand(platformModel, options);
|
|
7405
|
+
});
|
|
7406
|
+
sync.command("list [platform]").alias("ls").description("List all configured sync profiles and their status").action(async (platform) => {
|
|
7407
|
+
await syncListCommand(platform);
|
|
7408
|
+
});
|
|
7409
|
+
const schedule = sync.command("schedule").description("Manage scheduled syncs (runs via system cron on macOS/Linux)");
|
|
7410
|
+
schedule.command("add <platform>").description("Schedule a sync to run on an interval (e.g. --every 1h)").requiredOption("--every <duration>", "Interval: <n>m (divides 60), <n>h (divides 24), or 1d").option("--models <models>", "Only sync these models (comma-separated)").action(async (platform, options) => {
|
|
7411
|
+
await syncScheduleAddCommand(platform, options);
|
|
7412
|
+
});
|
|
7413
|
+
schedule.command("list").alias("ls").description("List all scheduled syncs").action(async () => {
|
|
7414
|
+
await syncScheduleListCommand();
|
|
7415
|
+
});
|
|
7416
|
+
schedule.command("remove <id-or-platform>").alias("rm").description("Remove a scheduled sync by id, or by platform (defaults to current directory)").option("--all", "When matching by platform, remove across all projects").action(async (idOrPlatform, options) => {
|
|
7417
|
+
await syncScheduleRemoveCommand(idOrPlatform, options);
|
|
7418
|
+
});
|
|
7419
|
+
schedule.command("status").description("Show scheduled syncs with last-run time, log tail, and drift detection").action(async () => {
|
|
7420
|
+
await syncScheduleStatusCommand();
|
|
7421
|
+
});
|
|
7422
|
+
schedule.command("repair <id>").description("Re-install a schedule whose cron line is missing or broken (drift)").action(async (id) => {
|
|
7423
|
+
await syncScheduleRepairCommand(id);
|
|
7424
|
+
});
|
|
7425
|
+
sync.command("remove <platform>").description("Remove sync data and profiles for a platform").option("--models <models>", "Specific models to remove (comma-separated)").option("--dry-run", "Show what would be removed without deleting").option("--yes", "Skip confirmation prompt").action(async (platform, options) => {
|
|
7426
|
+
await syncRemoveCommand(platform, options);
|
|
7427
|
+
});
|
|
7428
|
+
}
|
|
7429
|
+
|
|
7430
|
+
// src/commands/cache.ts
|
|
7431
|
+
import pc10 from "picocolors";
|
|
7432
|
+
async function cacheClearCommand(actionId) {
|
|
7433
|
+
if (actionId) {
|
|
7434
|
+
const deleted = clearEntry(actionId);
|
|
7435
|
+
if (isAgentMode()) {
|
|
7436
|
+
json({ cleared: deleted, actionId });
|
|
7437
|
+
return;
|
|
7438
|
+
}
|
|
7439
|
+
if (deleted) {
|
|
7440
|
+
console.log(`Cleared cache for ${pc10.cyan(actionId)}`);
|
|
7441
|
+
} else {
|
|
7442
|
+
console.log(`No cache entry found for ${pc10.dim(actionId)}`);
|
|
7443
|
+
}
|
|
7444
|
+
} else {
|
|
7445
|
+
const count = clearAll();
|
|
7446
|
+
if (isAgentMode()) {
|
|
7447
|
+
json({ cleared: true, count });
|
|
7448
|
+
return;
|
|
7449
|
+
}
|
|
7450
|
+
console.log(`Cleared ${count} cached ${count === 1 ? "entry" : "entries"}`);
|
|
7451
|
+
}
|
|
7452
|
+
}
|
|
7453
|
+
async function cacheListCommand(options) {
|
|
7454
|
+
const entries = listCacheEntries();
|
|
7455
|
+
const filtered = options.expired ? entries.filter((e) => !isFresh(e.entry)) : entries;
|
|
7456
|
+
if (isAgentMode()) {
|
|
7457
|
+
json({
|
|
7458
|
+
entries: filtered.map((e) => ({
|
|
7459
|
+
type: e.type,
|
|
7460
|
+
key: e.entry.key,
|
|
7461
|
+
cachedAt: e.entry.cachedAt,
|
|
7462
|
+
age: formatAge(getAge(e.entry)),
|
|
7463
|
+
ttl: e.entry.ttl,
|
|
7464
|
+
fresh: isFresh(e.entry),
|
|
7465
|
+
etag: e.entry.etag,
|
|
7466
|
+
path: e.filePath
|
|
7467
|
+
}))
|
|
7468
|
+
});
|
|
7469
|
+
return;
|
|
7470
|
+
}
|
|
7471
|
+
if (filtered.length === 0) {
|
|
7472
|
+
console.log(options.expired ? "No expired cache entries" : "No cached entries");
|
|
7473
|
+
return;
|
|
7474
|
+
}
|
|
7475
|
+
const rows = filtered.map((e) => ({
|
|
7476
|
+
type: e.type,
|
|
7477
|
+
key: e.entry.key,
|
|
7478
|
+
age: formatAge(getAge(e.entry)),
|
|
7479
|
+
status: isFresh(e.entry) ? pc10.green("fresh") : pc10.yellow("expired")
|
|
7480
|
+
}));
|
|
7481
|
+
printTable(
|
|
7482
|
+
[
|
|
7483
|
+
{ key: "type", label: "Type" },
|
|
7484
|
+
{ key: "key", label: "Key" },
|
|
7485
|
+
{ key: "age", label: "Age" },
|
|
7486
|
+
{ key: "status", label: "Status" }
|
|
7487
|
+
],
|
|
7488
|
+
rows
|
|
7489
|
+
);
|
|
7490
|
+
}
|
|
7491
|
+
async function cacheUpdateAllCommand() {
|
|
7492
|
+
const apiKey = getApiKey();
|
|
7493
|
+
if (!apiKey) {
|
|
7494
|
+
error("Not configured. Run `one init` first.");
|
|
7495
|
+
}
|
|
7496
|
+
const api = new OneApi(apiKey, getApiBase());
|
|
7497
|
+
const entries = listCacheEntries();
|
|
7498
|
+
if (entries.length === 0) {
|
|
7499
|
+
if (isAgentMode()) {
|
|
7500
|
+
json({ updated: 0, failed: 0, entries: [] });
|
|
7501
|
+
return;
|
|
7502
|
+
}
|
|
7503
|
+
console.log("No cached entries to update");
|
|
7504
|
+
return;
|
|
7505
|
+
}
|
|
7506
|
+
const spinner5 = createSpinner();
|
|
7507
|
+
spinner5.start(`Updating ${entries.length} cached ${entries.length === 1 ? "entry" : "entries"}...`);
|
|
7508
|
+
let updated = 0;
|
|
7509
|
+
let failed = 0;
|
|
7510
|
+
const errors = [];
|
|
7511
|
+
for (const e of entries) {
|
|
7512
|
+
try {
|
|
7513
|
+
if (e.type === "knowledge") {
|
|
7514
|
+
const result = await api.getActionKnowledgeWithMeta(e.entry.key);
|
|
7515
|
+
const newEntry = makeCacheEntry(e.entry.key, result.data, result.etag);
|
|
7516
|
+
writeCache2(e.filePath, newEntry);
|
|
7517
|
+
updated++;
|
|
7518
|
+
} else {
|
|
7519
|
+
const refreshed = { ...e.entry, cachedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
7520
|
+
writeCache2(e.filePath, refreshed);
|
|
7521
|
+
updated++;
|
|
7522
|
+
}
|
|
7523
|
+
} catch (err) {
|
|
7524
|
+
failed++;
|
|
7525
|
+
errors.push({
|
|
7526
|
+
key: e.entry.key,
|
|
7527
|
+
error: err instanceof Error ? err.message : "Unknown error"
|
|
7528
|
+
});
|
|
7529
|
+
}
|
|
7530
|
+
}
|
|
7531
|
+
spinner5.stop(`Updated ${updated} ${updated === 1 ? "entry" : "entries"}${failed > 0 ? `, ${failed} failed` : ""}`);
|
|
7532
|
+
if (isAgentMode()) {
|
|
7533
|
+
json({ updated, failed, errors: errors.length > 0 ? errors : void 0 });
|
|
7534
|
+
return;
|
|
7535
|
+
}
|
|
7536
|
+
if (errors.length > 0) {
|
|
7537
|
+
console.log();
|
|
7538
|
+
for (const e of errors) {
|
|
7539
|
+
console.log(` ${pc10.red("\u2717")} ${e.key}: ${pc10.dim(e.error)}`);
|
|
7540
|
+
}
|
|
7541
|
+
}
|
|
7542
|
+
}
|
|
7543
|
+
|
|
7544
|
+
// src/commands/guide.ts
|
|
7545
|
+
import pc11 from "picocolors";
|
|
7546
|
+
|
|
7547
|
+
// src/lib/guide-content.ts
|
|
7548
|
+
var GUIDE_OVERVIEW = `# One CLI \u2014 Agent Guide
|
|
7549
|
+
|
|
7550
|
+
## Setup
|
|
7551
|
+
|
|
7552
|
+
1. Run \`one init\` to configure your API key (interactive \u2014 can be global or per-project)
|
|
7553
|
+
2. Run \`one add <platform>\` to connect platforms via OAuth
|
|
7554
|
+
3. Run \`one --agent connection list\` to verify connections
|
|
7555
|
+
|
|
7556
|
+
## The --agent Flag
|
|
7557
|
+
|
|
7558
|
+
Always use \`--agent\` for machine-readable JSON output. It disables colors, spinners, and interactive prompts.
|
|
7559
|
+
|
|
7560
|
+
\`\`\`bash
|
|
7561
|
+
one --agent <command>
|
|
4035
7562
|
\`\`\`
|
|
4036
7563
|
|
|
4037
7564
|
All commands return JSON. If an \`error\` key is present, the command failed.
|
|
@@ -4060,6 +7587,10 @@ one --agent actions execute <platform> <actionId> <key> -d '{}' # Execute it
|
|
|
4060
7587
|
- \`--query-params <json>\` \u2014 Query parameters (arrays expand to repeated params)
|
|
4061
7588
|
- \`--form-url-encoded\` \u2014 Send as form data instead of JSON
|
|
4062
7589
|
- \`--dry-run\` \u2014 Preview request without executing
|
|
7590
|
+
- \`--mock\` \u2014 Return example response without making an API call (useful for building UI against a response shape)
|
|
7591
|
+
- \`--skip-validation\` \u2014 Skip input validation against the action schema
|
|
7592
|
+
|
|
7593
|
+
The CLI validates required parameters against the action schema before executing. If you're missing a required path variable, query param, or body field, you'll get a clear error listing what's missing and which flag to use. Pass \`--skip-validation\` to bypass.
|
|
4063
7594
|
|
|
4064
7595
|
Do NOT pass path or query parameters in \`-d\` \u2014 use the correct flags.
|
|
4065
7596
|
|
|
@@ -4080,7 +7611,8 @@ one --agent flow list # List all workflows
|
|
|
4080
7611
|
- 12 step types: action, transform, code, condition, loop, parallel, file-read, file-write, while, flow, paginate, bash
|
|
4081
7612
|
- Data wiring via selectors: \`$.input.param\`, \`$.steps.stepId.response\`, \`$.loop.item\`
|
|
4082
7613
|
- AI analysis via bash steps: \`claude --print\` with \`parseJson: true\`
|
|
4083
|
-
- Use \`--allow-bash\` to enable bash steps, \`--mock\` for dry-run with mock responses
|
|
7614
|
+
- Use \`--allow-bash\` to enable bash steps, \`--mock\` for dry-run with realistic mock responses (uses example data from action schemas)
|
|
7615
|
+
- Use \`--skip-validation\` to bypass input validation on action steps
|
|
4084
7616
|
|
|
4085
7617
|
### 3. Relay \u2014 Webhook event forwarding between platforms
|
|
4086
7618
|
Receive webhooks from platforms (Stripe, GitHub, Airtable, Attio, Google Calendar) and forward event data to any connected platform using passthrough actions with Handlebars templates. No middleware, no code.
|
|
@@ -4101,6 +7633,9 @@ one --agent relay deliveries --endpoint-id <id> # Check delivery
|
|
|
4101
7633
|
- \`--create-webhook\` auto-registers the webhook URL with the source platform
|
|
4102
7634
|
- Use \`actions knowledge\` to learn both the incoming payload shape AND the destination API shape before building templates
|
|
4103
7635
|
|
|
7636
|
+
### 4. Sync \u2014 Local data sync for instant offline queries
|
|
7637
|
+
Sync platform data into local SQLite for instant queries, full-text search, and change-driven automation. Requires a one-time \`one sync install\`. Run \`one guide sync\` for the full reference.
|
|
7638
|
+
|
|
4104
7639
|
## Topics
|
|
4105
7640
|
|
|
4106
7641
|
Request specific sections:
|
|
@@ -4109,6 +7644,7 @@ Request specific sections:
|
|
|
4109
7644
|
- \`one guide flows\` \u2014 Workflow engine reference (step types, selectors, examples)
|
|
4110
7645
|
- \`one guide relay\` \u2014 Webhook relay reference (templates, passthrough actions)
|
|
4111
7646
|
- \`one guide cache\` \u2014 Cache management (TTL, flags, commands)
|
|
7647
|
+
- \`one guide sync\` \u2014 Data sync reference (profiles, pagination, queries)
|
|
4112
7648
|
- \`one guide all\` \u2014 Everything
|
|
4113
7649
|
|
|
4114
7650
|
## Important Notes
|
|
@@ -4174,9 +7710,38 @@ one --agent actions execute <platform> <actionId> <connectionKey> [options]
|
|
|
4174
7710
|
- \`--headers <json>\` \u2014 Additional headers
|
|
4175
7711
|
- \`--form-data\` / \`--form-url-encoded\` \u2014 Alternative content types
|
|
4176
7712
|
- \`--dry-run\` \u2014 Preview without executing
|
|
7713
|
+
- \`--mock\` \u2014 Return example response without making an API call
|
|
7714
|
+
- \`--skip-validation\` \u2014 Skip input validation against the action schema
|
|
4177
7715
|
|
|
4178
7716
|
**Do NOT** pass path or query parameters in \`-d\`. Use the correct flags.
|
|
4179
7717
|
|
|
7718
|
+
### 4b. Parallel Execute
|
|
7719
|
+
|
|
7720
|
+
Execute multiple actions concurrently in a single command:
|
|
7721
|
+
|
|
7722
|
+
\`\`\`bash
|
|
7723
|
+
one --agent actions execute --parallel \\
|
|
7724
|
+
gmail send-email conn123 -d '{"to":"a@b.com"}' \\
|
|
7725
|
+
-- slack post-message conn456 -d '{"text":"done"}' \\
|
|
7726
|
+
-- google-sheets append-row conn789 -d '{"values":["x"]}'
|
|
7727
|
+
\`\`\`
|
|
7728
|
+
|
|
7729
|
+
Each segment separated by \`--\` follows the same format: \`<platform> <actionId> <connectionKey> [-d ...] [--path-vars ...] [--query-params ...]\`. Global flags (\`--dry-run\`, \`--mock\`, \`--skip-validation\`) apply to all segments.
|
|
7730
|
+
|
|
7731
|
+
All segments are validated upfront before any execution starts \u2014 if one segment has bad params, nothing runs. Execution uses \`Promise.allSettled\` so if one action fails the rest still complete. Use \`--max-concurrency <n>\` (default 5) to control batch size.
|
|
7732
|
+
|
|
7733
|
+
Agent-mode output:
|
|
7734
|
+
\`\`\`json
|
|
7735
|
+
{"parallel":true,"totalDurationMs":1234,"succeeded":2,"failed":0,"results":[{"segment":1,"platform":"gmail","actionId":"send-email","status":"success","durationMs":800,"response":{...}},{"segment":2,"platform":"slack","actionId":"post-message","status":"success","durationMs":600,"response":{...}}]}
|
|
7736
|
+
\`\`\`
|
|
7737
|
+
|
|
7738
|
+
## Input Validation
|
|
7739
|
+
|
|
7740
|
+
The CLI validates required parameters before executing. Missing params return a structured error:
|
|
7741
|
+
\`\`\`json
|
|
7742
|
+
{"error": "Validation failed: missing required parameters", "validation": {"missing": [{"flag": "--path-vars", "param": "userId", "description": "..."}]}, "hint": "...pass --skip-validation to bypass..."}
|
|
7743
|
+
\`\`\`
|
|
7744
|
+
|
|
4180
7745
|
## Error Handling
|
|
4181
7746
|
|
|
4182
7747
|
All errors return JSON: \`{"error": "message"}\`. Check the \`error\` key.
|
|
@@ -4350,12 +7915,282 @@ All cache commands respect \`--agent\` for JSON output.
|
|
|
4350
7915
|
| TTL (seconds) | \`cacheTtl\` in \`~/.one/config.json\` | \`"cacheTtl": 7200\` |
|
|
4351
7916
|
| Default | \u2014 | 3600 (1 hour) |
|
|
4352
7917
|
`;
|
|
7918
|
+
var GUIDE_SYNC = `# One Sync \u2014 Reference
|
|
7919
|
+
|
|
7920
|
+
Sync platform data into local SQLite for instant queries, full-text search, scheduled refresh, and change-driven automation.
|
|
7921
|
+
|
|
7922
|
+
## Getting Started
|
|
7923
|
+
|
|
7924
|
+
\`\`\`bash
|
|
7925
|
+
one sync install # One-time: install the SQLite engine
|
|
7926
|
+
one sync doctor # Verify it's working
|
|
7927
|
+
\`\`\`
|
|
7928
|
+
|
|
7929
|
+
## Built-in Profiles
|
|
7930
|
+
|
|
7931
|
+
Pre-validated sync configs ship with the CLI for common platforms. Discover them:
|
|
7932
|
+
|
|
7933
|
+
\`\`\`bash
|
|
7934
|
+
one --agent sync profiles # list all built-in profiles
|
|
7935
|
+
one --agent sync profiles stripe # filter by platform
|
|
7936
|
+
\`\`\`
|
|
7937
|
+
|
|
7938
|
+
When a built-in exists, \`sync init\` uses it automatically \u2014 no inference needed, no manual config. The agent just needs to match the user's intent to a profile description.
|
|
7939
|
+
|
|
7940
|
+
## Action Resolution
|
|
7941
|
+
|
|
7942
|
+
Sync profiles MUST prefer passthrough actions over custom actions.
|
|
7943
|
+
Custom actions add server-side fan-out and transformation that causes timeouts
|
|
7944
|
+
and payload size failures at scale. The sync engine handles pagination, retries,
|
|
7945
|
+
rate limiting, and enrichment locally \u2014 a server-side middleware layer on top
|
|
7946
|
+
of that creates problems, not value.
|
|
7947
|
+
|
|
7948
|
+
When resolving actions for sync profiles:
|
|
7949
|
+
1. Search with knowledge mode (not execute mode) to include passthrough actions
|
|
7950
|
+
2. Prefer GET passthrough endpoints (e.g. /gmail/v1/users/{userId}/threads)
|
|
7951
|
+
over POST custom endpoints (e.g. /gmail/get-threads)
|
|
7952
|
+
3. Use enrich config for per-record detail fetching instead of relying on
|
|
7953
|
+
custom actions that fan out server-side
|
|
7954
|
+
4. Only fall back to custom actions when no passthrough equivalent exists
|
|
7955
|
+
|
|
7956
|
+
This applies to sync models discovery, sync init, and enrich action selection.
|
|
7957
|
+
|
|
7958
|
+
## Workflow: init \u2192 run \u2192 query
|
|
7959
|
+
|
|
7960
|
+
\`\`\`bash
|
|
7961
|
+
# 1. Discover models
|
|
7962
|
+
one --agent sync models stripe
|
|
7963
|
+
|
|
7964
|
+
# 2. Init \u2014 one command does everything:
|
|
7965
|
+
# - resolves action ID
|
|
7966
|
+
# - infers pagination, resultsPath, idField, pathVars from knowledge
|
|
7967
|
+
# - auto-resolves connectionKey (when only one connection exists)
|
|
7968
|
+
# - auto-runs sync test if profile is complete
|
|
7969
|
+
one --agent sync init stripe balanceTransactions
|
|
7970
|
+
# Response includes _complete:true and _test results when fully resolved.
|
|
7971
|
+
# If connectionKey wasn't auto-resolved (multiple connections), patch it:
|
|
7972
|
+
one --agent sync init stripe balanceTransactions --config '{"connectionKey":"<from one list>"}'
|
|
7973
|
+
|
|
7974
|
+
# 3. Sync
|
|
7975
|
+
one --agent sync run stripe
|
|
7976
|
+
|
|
7977
|
+
# 4. Query
|
|
7978
|
+
one --agent sync query stripe/balanceTransactions --where "status=available" --limit 20
|
|
7979
|
+
one --agent sync search "refund" --platform stripe
|
|
7980
|
+
one --agent sync sql stripe "SELECT count(*) FROM balanceTransactions"
|
|
7981
|
+
\`\`\`
|
|
7982
|
+
|
|
7983
|
+
## Auto-Inference
|
|
7984
|
+
|
|
7985
|
+
\`sync init\` without \`--config\` does all of this automatically:
|
|
7986
|
+
- **connectionKey** \u2014 auto-resolved when there's exactly one connection for the platform
|
|
7987
|
+
- **Pagination** \u2014 Stripe id-pagination, Notion body-cursor, HubSpot/Google token, offset, link. Inapplicable fields stripped (no nextPath for offset, no passAs for none)
|
|
7988
|
+
- **resultsPath** \u2014 generic keys (data, results, items) + platform-specific (model name stripped of platform prefix: attioCompanies \u2192 companies)
|
|
7989
|
+
- **idField** \u2014 id, _id, uuid
|
|
7990
|
+
- **pathVars** \u2014 extracted from URL template with smart defaults (calendarId="primary", userId="me"). Internal keys (INTERNAL_SIGNING_KEY) and record-level IDs (record_id) are stripped automatically
|
|
7991
|
+
- **dateFilter** \u2014 updated_since, created_after, etc.
|
|
7992
|
+
- **limitLocation** \u2014 auto-detected as "body" for POST endpoints
|
|
7993
|
+
|
|
7994
|
+
When the profile is complete (no FILL_IN values remain), \`sync init\` automatically runs \`sync test\` and includes the results in the response (\`_test: {ok, checks, autoFixed}\`). If test auto-discovers fields the inference missed, it patches the profile on disk.
|
|
7995
|
+
|
|
7996
|
+
## Scheduled Syncs
|
|
7997
|
+
|
|
7998
|
+
\`\`\`bash
|
|
7999
|
+
one sync schedule add stripe --every 1h
|
|
8000
|
+
one sync schedule add notion --every 30m --models search
|
|
8001
|
+
one --agent sync schedule list # Works from any directory
|
|
8002
|
+
one --agent sync schedule status # Drift detection + log tails
|
|
8003
|
+
one sync schedule remove <id|platform> # By id (any dir) or platform
|
|
8004
|
+
one sync schedule repair <id> # Re-install broken cron line
|
|
8005
|
+
\`\`\`
|
|
8006
|
+
Backed by system cron (macOS/Linux). Schedules tracked in a global registry at \`~/.one/sync/schedules.json\`.
|
|
8007
|
+
|
|
8008
|
+
## Record Enrichment
|
|
8009
|
+
|
|
8010
|
+
When a list endpoint returns lightweight records (e.g. just IDs), add an \`enrich\` config to call a detail endpoint per record and merge the full data before storing:
|
|
8011
|
+
|
|
8012
|
+
\`\`\`json
|
|
8013
|
+
{
|
|
8014
|
+
"enrich": {
|
|
8015
|
+
"actionId": "<get-message-action-id>",
|
|
8016
|
+
"pathVars": { "messageId": "{{id}}" },
|
|
8017
|
+
"concurrency": 3,
|
|
8018
|
+
"delayMs": 200
|
|
8019
|
+
}
|
|
8020
|
+
}
|
|
8021
|
+
\`\`\`
|
|
8022
|
+
|
|
8023
|
+
- \`pathVars\` / \`queryParams\` / \`body\` support \`{{field}}\` interpolation from the list record
|
|
8024
|
+
- \`concurrency\` controls parallel detail requests per page (default: 3, lower = safer for rate limits)
|
|
8025
|
+
- \`delayMs\` is the pause between batches (default: 200ms)
|
|
8026
|
+
- \`resultsPath\` extracts a sub-object from the detail response before merging
|
|
8027
|
+
- \`merge: false\` replaces the record entirely instead of deep-merging
|
|
8028
|
+
|
|
8029
|
+
**Rate limiting is first-class:**
|
|
8030
|
+
- Honors \`Retry-After\` headers from 429 responses
|
|
8031
|
+
- Exponential backoff (2s \u2192 4s \u2192 8s)
|
|
8032
|
+
- Adaptive throttle: if any request in a batch hits 429, concurrency halves automatically
|
|
8033
|
+
- Records that fail after 3 retries are skipped (sync continues, count reported in \`enrichSkipped\`)
|
|
8034
|
+
|
|
8035
|
+
**Important:** \`enrich.resultsPath\` operates on the raw API response, NOT the CLI's \`{dryRun, request, response}\` wrapper you see when testing with \`one --agent actions execute\`. If the CLI shows your data at \`response.thread\`, the enrich resultsPath is just \`"thread"\` (no \`response.\` prefix).
|
|
8036
|
+
|
|
8037
|
+
Enrichment runs after list sync completes (Phase 2), not inline. It's inherently resumable \u2014 records track an \`_enriched_at\` timestamp, and re-running skips already-enriched rows.
|
|
8038
|
+
|
|
8039
|
+
**Limitation:** Each profile supports one enrich action. If you need multiple enrichments (e.g. both summary and transcript from Fathom), create a second profile/model for the second enrichment.
|
|
8040
|
+
|
|
8041
|
+
## Record Transform
|
|
8042
|
+
|
|
8043
|
+
Pipe records through any shell command or flow between fetch and store. The command receives a JSON array on stdin and must return a JSON array on stdout.
|
|
8044
|
+
|
|
8045
|
+
\`\`\`json
|
|
8046
|
+
{
|
|
8047
|
+
"transform": "jq '[.[] | . + {flat_title: (.properties.title.title[0].plain_text // null)}]'"
|
|
8048
|
+
}
|
|
8049
|
+
\`\`\`
|
|
8050
|
+
|
|
8051
|
+
Use cases:
|
|
8052
|
+
- Flatten nested fields into queryable top-level columns
|
|
8053
|
+
- Add computed fields (tags, categories, scores)
|
|
8054
|
+
- Filter out records you don't want to store
|
|
8055
|
+
- Reshape API responses into a cleaner schema
|
|
8056
|
+
|
|
8057
|
+
The transform can be any command: \`jq\`, \`python3\`, a bash script, or \`one flow execute <key>\`. If the command fails, times out (60s), or returns invalid JSON, the original records are used (warning printed, sync continues).
|
|
8058
|
+
|
|
8059
|
+
**Pipeline order:** fetch \u2192 enrich \u2192 transform \u2192 **exclude** \u2192 create table \u2192 schema evolution \u2192 upsert \u2192 hooks
|
|
8060
|
+
|
|
8061
|
+
## Cross-Platform Identity
|
|
8062
|
+
|
|
8063
|
+
Add \`identityKey\` to a sync profile to extract a stable cross-platform identifier (e.g. email) into a normalized \`_identity\` column:
|
|
8064
|
+
|
|
8065
|
+
\`\`\`json
|
|
8066
|
+
{"platform": "hubspot", "model": "contacts", "identityKey": "properties.email"}
|
|
8067
|
+
{"platform": "stripe", "model": "customers", "identityKey": "email"}
|
|
8068
|
+
{"platform": "attio", "model": "attioPeople", "identityKey": "email_addresses[0].email_address"}
|
|
8069
|
+
\`\`\`
|
|
8070
|
+
|
|
8071
|
+
The value is lowercased and trimmed. Query across platforms:
|
|
8072
|
+
\`\`\`bash
|
|
8073
|
+
one --agent sync sql hubspot "SELECT * FROM contacts WHERE _identity = 'jane@acme.com'"
|
|
8074
|
+
one --agent sync sql stripe "SELECT * FROM customers WHERE _identity = 'jane@acme.com'"
|
|
8075
|
+
\`\`\`
|
|
8076
|
+
|
|
8077
|
+
## Exclude Fields
|
|
8078
|
+
|
|
8079
|
+
Strip large or unwanted fields from records before storing (e.g. base64 attachments, raw HTML bodies):
|
|
8080
|
+
|
|
8081
|
+
\`\`\`json
|
|
8082
|
+
{ "exclude": ["messages[].body", "messages[].attachments[].data", "payload.parts"] }
|
|
8083
|
+
\`\`\`
|
|
8084
|
+
|
|
8085
|
+
Supports dot-path notation and array iteration (\`messages[].body\` strips \`body\` from each element of the \`messages\` array). Runs before table creation so excluded columns never exist in the schema.
|
|
8086
|
+
|
|
8087
|
+
## Monitoring Progress
|
|
8088
|
+
|
|
8089
|
+
\`sync list\` doubles as a progress monitor. The state file is updated after every page, so while a sync is running you can check progress from another context:
|
|
8090
|
+
|
|
8091
|
+
\`\`\`bash
|
|
8092
|
+
one --agent sync list gmail
|
|
8093
|
+
# \u2192 {"syncs":[{"model":"gmailThreads","totalRecords":400,"pagesProcessed":8,"status":"syncing",...}]}
|
|
8094
|
+
\`\`\`
|
|
8095
|
+
|
|
8096
|
+
When \`status\` is \`"syncing"\`, \`totalRecords\` and \`pagesProcessed\` reflect real-time progress. When it flips to \`"idle"\`, the sync is done. No need to babysit \u2014 especially when using \`sync schedule\` for unattended runs.
|
|
8097
|
+
|
|
8098
|
+
## Change Hooks (CDC)
|
|
8099
|
+
|
|
8100
|
+
Add \`onInsert\`, \`onUpdate\`, or \`onChange\` to a sync profile to fire hooks when records change:
|
|
8101
|
+
|
|
8102
|
+
\`\`\`json
|
|
8103
|
+
{
|
|
8104
|
+
"onInsert": "one flow execute enrich-new-contact",
|
|
8105
|
+
"onUpdate": "log",
|
|
8106
|
+
"onChange": "node ./scripts/handle-change.js"
|
|
8107
|
+
}
|
|
8108
|
+
\`\`\`
|
|
8109
|
+
|
|
8110
|
+
**Hook modes:**
|
|
8111
|
+
- **Shell command** \u2014 record events piped as NDJSON to stdin
|
|
8112
|
+
- **\`"log"\`** \u2014 append to \`.one/sync/events/<platform>_<model>.jsonl\`
|
|
8113
|
+
- **Flow execution** \u2014 \`one flow execute <key>\` with record as input
|
|
8114
|
+
|
|
8115
|
+
Hooks fire after each page (not end-of-sync) for real-time processing. Each event:
|
|
8116
|
+
\`{"type":"insert|update","platform":"...","model":"...","record":{...},"timestamp":"..."}\`
|
|
8117
|
+
|
|
8118
|
+
Every record has a \`_synced_at\` timestamp so you can track when it was last pulled.
|
|
8119
|
+
|
|
8120
|
+
## Full Refresh (deletion detection)
|
|
8121
|
+
|
|
8122
|
+
\`\`\`bash
|
|
8123
|
+
one --agent sync run stripe --full-refresh
|
|
8124
|
+
\`\`\`
|
|
8125
|
+
Fetches ALL records and deletes local rows whose IDs are no longer in the source. Cannot be combined with \`--since\`.
|
|
8126
|
+
|
|
8127
|
+
## Commands Reference
|
|
8128
|
+
|
|
8129
|
+
| Command | What it does |
|
|
8130
|
+
|---------|-------------|
|
|
8131
|
+
| \`sync profiles [platform]\` | List built-in pre-validated profiles |
|
|
8132
|
+
| \`sync install\` | Install SQLite engine (first time) |
|
|
8133
|
+
| \`sync doctor\` | Verify engine health |
|
|
8134
|
+
| \`sync models <platform>\` | Discover available models |
|
|
8135
|
+
| \`sync init <plat> <model>\` | Create profile (auto-infers from knowledge) |
|
|
8136
|
+
| \`sync test <plat>/<model>\` | Validate profile + auto-fix fields |
|
|
8137
|
+
| \`sync run <platform>\` | Sync data (\`--full-refresh\`, \`--since\`, \`--dry-run\`) |
|
|
8138
|
+
| \`sync query <plat>/<model>\` | Query with \`--where\`, \`--after/before\`, \`--refresh\` |
|
|
8139
|
+
| \`sync search "<query>"\` | FTS5 across all synced data |
|
|
8140
|
+
| \`sync sql <plat> "<sql>"\` | Raw SELECT queries |
|
|
8141
|
+
| \`sync list [platform]\` | Show profiles, record counts, freshness |
|
|
8142
|
+
| \`sync schedule add/list/status/remove/repair\` | Manage cron schedules |
|
|
8143
|
+
| \`sync remove <platform>\` | Delete local data (\`--dry-run\` to preview) |
|
|
8144
|
+
|
|
8145
|
+
## Sync Profile Fields
|
|
8146
|
+
|
|
8147
|
+
| Field | Required | Description |
|
|
8148
|
+
|-------|----------|-------------|
|
|
8149
|
+
| connectionKey | yes | From \`one list\` |
|
|
8150
|
+
| actionId | yes | Auto-resolved by \`sync init\` |
|
|
8151
|
+
| resultsPath | yes | Auto-inferred or auto-discovered by \`sync test\` |
|
|
8152
|
+
| idField | yes | Auto-inferred or auto-discovered by \`sync test\` |
|
|
8153
|
+
| pagination | yes | Auto-inferred (cursor/token/offset/id/link/none) |
|
|
8154
|
+
| pathVars | no | Auto-extracted from URL template |
|
|
8155
|
+
| dateFilter | no | For incremental sync (auto-detected when available) |
|
|
8156
|
+
| limitParam | no | Page size param name (empty string = don't send) |
|
|
8157
|
+
| limitLocation | no | "query" (default) or "body" for POST endpoints |
|
|
8158
|
+
| enrich | no | Detail endpoint config for record enrichment (actionId, pathVars, concurrency) |
|
|
8159
|
+
| transform | no | Shell command to transform records (stdin: JSON array, stdout: JSON array) |
|
|
8160
|
+
| identityKey | no | Dot-path to cross-platform identifier (e.g. email) \u2192 stored as \`_identity\` column |
|
|
8161
|
+
| exclude | no | Dot-path fields to strip before storing (e.g. \`["messages[].body"]\`) |
|
|
8162
|
+
| onInsert/onUpdate/onChange | no | Change hooks (shell command, "log", or flow) |
|
|
8163
|
+
|
|
8164
|
+
## Pagination Types
|
|
8165
|
+
|
|
8166
|
+
- **cursor** \u2014 \`{"type":"cursor", "nextPath":"next_cursor", "passAs":"query:cursor"}\`
|
|
8167
|
+
- **token** \u2014 \`{"type":"token", "nextPath":"paging.next.after", "passAs":"query:after"}\`
|
|
8168
|
+
- **offset** \u2014 \`{"type":"offset", "passAs":"query:offset", "totalPath":"total"}\`
|
|
8169
|
+
- **id** \u2014 \`{"type":"id", "passAs":"query:starting_after", "hasMorePath":"has_more"}\`
|
|
8170
|
+
- **link** \u2014 \`{"type":"link", "nextPath":"link.next.page_info", "passAs":"query:page_info"}\`
|
|
8171
|
+
- **none** \u2014 single request, no pagination (no limit param injected)
|
|
8172
|
+
|
|
8173
|
+
## File Layout
|
|
8174
|
+
|
|
8175
|
+
\`\`\`
|
|
8176
|
+
.one/sync/
|
|
8177
|
+
profiles/{platform}_{model}.json # sync profiles
|
|
8178
|
+
data/{platform}.db # SQLite databases (WAL mode)
|
|
8179
|
+
sync_state.json # checkpoint tracking
|
|
8180
|
+
events/{platform}_{model}.jsonl # change event logs (if onChange: "log")
|
|
8181
|
+
logs/{platform}.log # cron run logs
|
|
8182
|
+
locks/{platform}_{model}/ # cross-process sync locks
|
|
8183
|
+
~/.one/sync/
|
|
8184
|
+
schedules.json # global schedule registry
|
|
8185
|
+
\`\`\`
|
|
8186
|
+
`;
|
|
4353
8187
|
var TOPICS = [
|
|
4354
8188
|
{ topic: "overview", description: "Setup, features, and quick start for each" },
|
|
4355
8189
|
{ topic: "actions", description: "Search, read docs, and execute platform actions" },
|
|
4356
8190
|
{ topic: "flows", description: "Build and execute multi-step workflows" },
|
|
4357
8191
|
{ topic: "relay", description: "Receive webhooks and forward to other platforms" },
|
|
4358
8192
|
{ topic: "cache", description: "Local caching for knowledge and search responses" },
|
|
8193
|
+
{ topic: "sync", description: "Sync platform data locally for instant offline queries" },
|
|
4359
8194
|
{ topic: "all", description: "Complete guide (all topics combined)" }
|
|
4360
8195
|
];
|
|
4361
8196
|
function getGuideContent(topic) {
|
|
@@ -4370,10 +8205,12 @@ function getGuideContent(topic) {
|
|
|
4370
8205
|
return { title: "One CLI \u2014 Agent Guide: Relay", content: GUIDE_RELAY };
|
|
4371
8206
|
case "cache":
|
|
4372
8207
|
return { title: "One CLI \u2014 Agent Guide: Cache", content: GUIDE_CACHE };
|
|
8208
|
+
case "sync":
|
|
8209
|
+
return { title: "One CLI \u2014 Agent Guide: Sync", content: GUIDE_SYNC };
|
|
4373
8210
|
case "all":
|
|
4374
8211
|
return {
|
|
4375
8212
|
title: "One CLI \u2014 Agent Guide: Complete",
|
|
4376
|
-
content: [GUIDE_OVERVIEW, GUIDE_ACTIONS, GUIDE_FLOWS, GUIDE_RELAY, GUIDE_CACHE].join("\n---\n\n")
|
|
8213
|
+
content: [GUIDE_OVERVIEW, GUIDE_ACTIONS, GUIDE_FLOWS, GUIDE_RELAY, GUIDE_CACHE, GUIDE_SYNC].join("\n---\n\n")
|
|
4377
8214
|
};
|
|
4378
8215
|
}
|
|
4379
8216
|
}
|
|
@@ -4382,7 +8219,7 @@ function getAvailableTopics() {
|
|
|
4382
8219
|
}
|
|
4383
8220
|
|
|
4384
8221
|
// src/commands/guide.ts
|
|
4385
|
-
var VALID_TOPICS = ["overview", "actions", "flows", "relay", "cache", "all"];
|
|
8222
|
+
var VALID_TOPICS = ["overview", "actions", "flows", "relay", "cache", "sync", "all"];
|
|
4386
8223
|
async function guideCommand(topic = "all") {
|
|
4387
8224
|
if (!VALID_TOPICS.includes(topic)) {
|
|
4388
8225
|
error(
|
|
@@ -4395,14 +8232,14 @@ async function guideCommand(topic = "all") {
|
|
|
4395
8232
|
json({ topic, title, content, availableTopics });
|
|
4396
8233
|
return;
|
|
4397
8234
|
}
|
|
4398
|
-
intro2(
|
|
8235
|
+
intro2(pc11.bgCyan(pc11.black(" One Guide ")));
|
|
4399
8236
|
console.log();
|
|
4400
8237
|
console.log(content);
|
|
4401
|
-
console.log(
|
|
8238
|
+
console.log(pc11.dim("\u2500".repeat(60)));
|
|
4402
8239
|
console.log(
|
|
4403
|
-
|
|
8240
|
+
pc11.dim("Available topics: ") + availableTopics.map((t) => pc11.cyan(t.topic)).join(", ")
|
|
4404
8241
|
);
|
|
4405
|
-
console.log(
|
|
8242
|
+
console.log(pc11.dim(`Run ${pc11.cyan("one guide <topic>")} for a specific section.`));
|
|
4406
8243
|
}
|
|
4407
8244
|
|
|
4408
8245
|
// src/lib/platform-meta.ts
|
|
@@ -4441,7 +8278,7 @@ function pairKey(a, b) {
|
|
|
4441
8278
|
}
|
|
4442
8279
|
function getWorkflowExamples(connectedPlatforms) {
|
|
4443
8280
|
const results = [];
|
|
4444
|
-
const platforms = connectedPlatforms.map((
|
|
8281
|
+
const platforms = connectedPlatforms.map((p8) => p8.toLowerCase());
|
|
4445
8282
|
for (let i = 0; i < platforms.length; i++) {
|
|
4446
8283
|
for (let j = i + 1; j < platforms.length; j++) {
|
|
4447
8284
|
const key = pairKey(platforms[i], platforms[j]);
|
|
@@ -4685,13 +8522,13 @@ function buildDemoActions(connections) {
|
|
|
4685
8522
|
const connectedPlatforms = connections.map((c) => c.platform.toLowerCase());
|
|
4686
8523
|
const popularPlatforms = ["gmail", "google-calendar", "slack", "shopify", "hub-spot", "github"];
|
|
4687
8524
|
const platformsToShow = [
|
|
4688
|
-
...connectedPlatforms.filter((
|
|
4689
|
-
...popularPlatforms.filter((
|
|
8525
|
+
...connectedPlatforms.filter((p8) => PLATFORM_DEMO_ACTIONS[p8]),
|
|
8526
|
+
...popularPlatforms.filter((p8) => !connectedPlatforms.includes(p8))
|
|
4690
8527
|
];
|
|
4691
8528
|
const seen = /* @__PURE__ */ new Set();
|
|
4692
|
-
const unique = platformsToShow.filter((
|
|
4693
|
-
if (seen.has(
|
|
4694
|
-
seen.add(
|
|
8529
|
+
const unique = platformsToShow.filter((p8) => {
|
|
8530
|
+
if (seen.has(p8)) return false;
|
|
8531
|
+
seen.add(p8);
|
|
4695
8532
|
return true;
|
|
4696
8533
|
}).slice(0, 6);
|
|
4697
8534
|
for (const platform of unique) {
|
|
@@ -4749,6 +8586,15 @@ program.name("one").option("--agent", "Machine-readable JSON output (no colors,
|
|
|
4749
8586
|
one flow execute <key> Execute a workflow
|
|
4750
8587
|
one flow validate <key> Validate a flow
|
|
4751
8588
|
|
|
8589
|
+
Data Sync (run "one sync install" first, then "one guide sync" for full reference):
|
|
8590
|
+
one sync models <platform> Discover available data models
|
|
8591
|
+
one sync init <plat> <model> Create profile (auto-infers from knowledge)
|
|
8592
|
+
one sync test <plat>/<model> Validate + auto-fix profile fields
|
|
8593
|
+
one sync run <platform> Sync data (--full-refresh for deletions)
|
|
8594
|
+
one sync query <plat>/<model> Query local data (--where, --refresh)
|
|
8595
|
+
one sync search "<query>" Full-text search across all synced data
|
|
8596
|
+
one sync schedule add <plat> --every Cron schedule (e.g. 1h) with change hooks
|
|
8597
|
+
|
|
4752
8598
|
Cache:
|
|
4753
8599
|
one cache list List cached entries with age and status
|
|
4754
8600
|
one cache clear Clear all cached knowledge and search data
|
|
@@ -4901,7 +8747,14 @@ actions.command("search <platform> <query>").description('Search for actions on
|
|
|
4901
8747
|
actions.command("knowledge <platform> <actionId>").alias("k").description("Get full docs for an action \u2014 MUST call before execute to know required params").option("--no-cache", "Skip cache, fetch fresh from API").option("--cache-status", "Print cache metadata without fetching").action(async (platform, actionId, options) => {
|
|
4902
8748
|
await actionsKnowledgeCommand(platform, actionId, options);
|
|
4903
8749
|
});
|
|
4904
|
-
actions.command("execute
|
|
8750
|
+
actions.command("execute [platform] [actionId] [connectionKey]").alias("x").allowUnknownOption(true).allowExcessArguments(true).description("Execute an action (or multiple with --parallel, separated by --)").option("-d, --data <json>", "Request body as JSON").option("--path-vars <json>", "Path variables as JSON").option("--query-params <json>", "Query parameters as JSON").option("--headers <json>", "Additional headers as JSON").option("--form-data", "Send as multipart/form-data").option("--form-url-encoded", "Send as application/x-www-form-urlencoded").option("--dry-run", "Show request that would be sent without executing").option("--mock", "Return example response without making an API call").option("--skip-validation", "Skip input validation against the action schema").option("--parallel", "Execute multiple actions concurrently (separate actions with --)").option("--max-concurrency <n>", "Max concurrent actions when using --parallel (default: 5)", "5").action(async (platform, actionId, connectionKey, options) => {
|
|
8751
|
+
if (options.parallel) {
|
|
8752
|
+
await actionsExecuteParallelCommand();
|
|
8753
|
+
return;
|
|
8754
|
+
}
|
|
8755
|
+
if (!platform || !actionId || !connectionKey) {
|
|
8756
|
+
error("Usage: one actions execute <platform> <actionId> <connectionKey> [-d ...]");
|
|
8757
|
+
}
|
|
4905
8758
|
await actionsExecuteCommand(platform, actionId, connectionKey, {
|
|
4906
8759
|
data: options.data,
|
|
4907
8760
|
pathVars: options.pathVars,
|
|
@@ -4909,14 +8762,16 @@ actions.command("execute <platform> <actionId> <connectionKey>").alias("x").desc
|
|
|
4909
8762
|
headers: options.headers,
|
|
4910
8763
|
formData: options.formData,
|
|
4911
8764
|
formUrlEncoded: options.formUrlEncoded,
|
|
4912
|
-
dryRun: options.dryRun
|
|
8765
|
+
dryRun: options.dryRun,
|
|
8766
|
+
mock: options.mock,
|
|
8767
|
+
skipValidation: options.skipValidation
|
|
4913
8768
|
});
|
|
4914
8769
|
});
|
|
4915
8770
|
var flow = program.command("flow").alias("f").description("Create, execute, and manage multi-step workflows");
|
|
4916
8771
|
flow.command("create [key]").description("Create a new workflow from JSON definition").option("--definition <json>", "Workflow definition as JSON string").option("-o, --output <path>", "Custom output path (default .one/flows/<key>/flow.json)").action(async (key, options) => {
|
|
4917
8772
|
await flowCreateCommand(key, options);
|
|
4918
8773
|
});
|
|
4919
|
-
flow.command("execute <keyOrPath>").alias("x").description("Execute a workflow by key or file path").option("-i, --input <name=value>", "Input parameter (repeatable)", collect, []).option("--dry-run", "Validate and show execution plan without running").option("--mock", "With --dry-run: execute transforms/code with mock API responses").option("--allow-bash", "Allow bash step execution (disabled by default for security)").option("-v, --verbose", "Show full request/response for each step").action(async (keyOrPath, options) => {
|
|
8774
|
+
flow.command("execute <keyOrPath>").alias("x").description("Execute a workflow by key or file path").option("-i, --input <name=value>", "Input parameter (repeatable)", collect, []).option("--dry-run", "Validate and show execution plan without running").option("--mock", "With --dry-run: execute transforms/code with realistic mock API responses").option("--skip-validation", "Skip input validation against action schemas").option("--allow-bash", "Allow bash step execution (disabled by default for security)").option("-v, --verbose", "Show full request/response for each step").action(async (keyOrPath, options) => {
|
|
4920
8775
|
await flowExecuteCommand(keyOrPath, options);
|
|
4921
8776
|
});
|
|
4922
8777
|
flow.command("list").alias("ls").description("List all workflows in .one/flows/").action(async () => {
|
|
@@ -4965,6 +8820,7 @@ relay.command("deliveries").description("List delivery attempts for an endpoint
|
|
|
4965
8820
|
relay.command("event-types <platform>").description("List supported webhook event types for a platform").action(async (platform) => {
|
|
4966
8821
|
await relayEventTypesCommand(platform);
|
|
4967
8822
|
});
|
|
8823
|
+
registerSyncCommands(program);
|
|
4968
8824
|
var cache = program.command("cache").description("Manage the local knowledge and search cache");
|
|
4969
8825
|
cache.command("clear [actionId]").description("Clear all cached data, or a specific action by ID").action(async (actionId) => {
|
|
4970
8826
|
await cacheClearCommand(actionId);
|
|
@@ -4975,7 +8831,7 @@ cache.command("list").alias("ls").description("List all cached entries with age
|
|
|
4975
8831
|
cache.command("update-all").description("Re-fetch fresh data for all cached entries").action(async () => {
|
|
4976
8832
|
await cacheUpdateAllCommand();
|
|
4977
8833
|
});
|
|
4978
|
-
program.command("guide [topic]").description("Full CLI usage guide for agents (topics: overview, actions, flows, relay, all)").action(async (topic) => {
|
|
8834
|
+
program.command("guide [topic]").description("Full CLI usage guide for agents (topics: overview, actions, flows, relay, cache, sync, all)").action(async (topic) => {
|
|
4979
8835
|
await guideCommand(topic);
|
|
4980
8836
|
});
|
|
4981
8837
|
program.command("onboard").description("Agent onboarding \u2014 teaches your agent what the One CLI can do").option("--step <number>", "Run a specific onboarding step (1, 2, or 3)").action(async (options) => {
|