@aiwg/cli 2026.10.11 → 2026.10.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agentic/code/providers/model-catalog.v1.json +62 -12
- package/agentic/code/providers/pi/aiwg-bridge.ts +44 -1
- package/dist/src/artifacts/binding-target-identity.js +51 -0
- package/dist/src/artifacts/checksum-manifest.js +12 -0
- package/dist/src/artifacts/index-builder.js +11 -13
- package/dist/src/artifacts/query-engine.js +5 -11
- package/dist/src/cli/handlers/doctor-help.js +32 -0
- package/dist/src/cli/handlers/gates.js +19 -18
- package/dist/src/cli/handlers/models-help.js +7 -1
- package/dist/src/cli/handlers/models.js +3 -3
- package/dist/src/cli/handlers/script-runner.js +8 -5
- package/dist/src/cli/handlers/sessions-intake.js +104 -0
- package/dist/src/cli/handlers/sessions-tail.js +172 -0
- package/dist/src/cli/handlers/sessions.js +90 -17
- package/dist/src/cli/handlers/skill-lint.js +10 -3
- package/dist/src/cli/handlers/steward-format.js +100 -0
- package/dist/src/cli/handlers/steward-models.js +167 -0
- package/dist/src/cli/handlers/steward.js +14 -248
- package/dist/src/cli/handlers/use-bundle-counts.js +87 -0
- package/dist/src/cli/handlers/use-skill-assets.js +207 -0
- package/dist/src/cli/handlers/use.js +35 -216
- package/dist/src/cli/handlers/utilities.js +1 -24
- package/dist/src/config/aiwg-config.js +7 -0
- package/dist/src/config/authorization-cli.js +97 -0
- package/dist/src/config/cli-help.js +1 -0
- package/dist/src/config/cli.js +10 -1
- package/dist/src/decision/context-pruning-common.js +41 -0
- package/dist/src/decision/context-pruning-evaluation.js +366 -0
- package/dist/src/decision/context-pruning-pilot.js +431 -0
- package/dist/src/decision/context-pruning-types.js +19 -0
- package/dist/src/decision/context-pruning.js +5 -846
- package/dist/src/decision/ensemble/contract.js +17 -26
- package/dist/src/decision/ensemble-study/artifacts.js +5 -0
- package/dist/src/decision/ensemble-study/score.js +4 -3
- package/dist/src/decision/feature-export.js +3 -9
- package/dist/src/decision/heldout/collector.js +2 -2
- package/dist/src/decision/issue-triage/runtime.js +3 -9
- package/dist/src/decision/qualification/gates.js +2 -1
- package/dist/src/decision/qualification/release.js +6 -64
- package/dist/src/decision/routing/report.js +2 -0
- package/dist/src/decision/sdlc-screening.js +3 -5
- package/dist/src/decision/sensitivity/comparative.js +5 -3
- package/dist/src/extensions/manifest.js +8 -1
- package/dist/src/extensions/project-local-doctor.js +78 -2
- package/dist/src/extensions/validation.js +10 -11
- package/dist/src/gates/driver.js +6 -77
- package/dist/src/gates/evaluate.js +6 -23
- package/dist/src/gates/floors.js +4 -4
- package/dist/src/gates/index.js +3 -3
- package/dist/src/gates/lattice.js +26 -0
- package/dist/src/gates/registry.js +2 -1
- package/dist/src/gates/report.js +19 -7
- package/dist/src/gates/upstream-ceiling.js +111 -0
- package/dist/src/issues/audit-cli.js +120 -0
- package/dist/src/issues/audit-contract.js +64 -0
- package/dist/src/issues/audit-local.js +82 -0
- package/dist/src/issues/audit.js +220 -0
- package/dist/src/lint/rule-activation.js +15 -1
- package/dist/src/mission-protocol/canonical-codecs.js +270 -0
- package/dist/src/mission-protocol/codecs.js +22 -550
- package/dist/src/mission-protocol/health-trace-codecs.js +281 -0
- package/dist/src/mission-protocol/health-trace-types.js +43 -0
- package/dist/src/mission-protocol/types.js +2 -42
- package/dist/src/mission-protocol/worker-contract-json.js +10 -0
- package/dist/src/mission-protocol/worker-review-gate.js +100 -0
- package/dist/src/mission-protocol/worker-review-types.js +2 -0
- package/dist/src/models/catalog-policy.js +280 -0
- package/dist/src/models/index.js +1 -1
- package/dist/src/models/model-catalog.v1.json +62 -12
- package/dist/src/models/policy-resources.js +28 -0
- package/dist/src/models/provider-models.js +5 -8
- package/dist/src/models/provider-policy.js +10 -546
- package/dist/src/models/resolver.js +26 -8
- package/dist/src/models/router.js +15 -2
- package/dist/src/models/tiers.js +13 -0
- package/dist/src/models/types.js +0 -6
- package/dist/src/models/worker-ceilings.js +59 -0
- package/dist/src/models/worker-invocation-policy.js +243 -0
- package/dist/src/models/worker-plan.js +16 -20
- package/dist/src/models/worker-review-lineage.js +55 -0
- package/dist/src/models/worker-selection-policy.js +1 -1
- package/dist/src/models/wrapper-definitions.js +14 -0
- package/dist/src/models/wrapper-deployment.js +48 -25
- package/dist/src/models/wrapper-enforcement.js +72 -0
- package/dist/src/models/wrapper-route.js +6 -3
- package/dist/src/policy/authorization.js +1 -0
- package/dist/src/policy/hitl-check.js +80 -0
- package/dist/src/policy/hitl-config.js +63 -0
- package/dist/src/policy/hitl-receipt.js +57 -0
- package/dist/src/policy/hitl-types.js +4 -0
- package/dist/src/policy/hitl-validation.js +39 -0
- package/dist/src/policy/hitl.js +117 -0
- package/dist/src/rules/activation.d.mts +3 -1
- package/dist/src/rules/activation.mjs +23 -2
- package/dist/src/rules/gate-coverage.js +51 -0
- package/dist/src/rules/gate-references.js +16 -0
- package/dist/src/rules/rule-frontmatter.schema.json +7 -1
- package/dist/src/serve/worker-command-request.js +21 -4
- package/dist/src/serve/worker-invocation-preparation.js +3 -1
- package/dist/src/serve/worker-invocation.js +21 -10
- package/dist/src/sessions/adapters/deepseek-harness.js +1 -0
- package/dist/src/sessions/adapters/grok-build-native.js +269 -0
- package/dist/src/sessions/adapters/grok-build.js +57 -15
- package/dist/src/sessions/adapters/hermes-state-db.js +252 -0
- package/dist/src/sessions/adapters/hermes.js +42 -3
- package/dist/src/sessions/adapters/warp.js +1 -1
- package/dist/src/sessions/batch-import.js +10 -0
- package/dist/src/sessions/contracts.js +1 -1
- package/dist/src/sessions/discovery-capability.js +38 -0
- package/dist/src/sessions/importer.js +8 -251
- package/dist/src/sessions/index.js +4 -0
- package/dist/src/sessions/intake-dir.js +71 -0
- package/dist/src/sessions/normalization.js +251 -0
- package/dist/src/sessions/tail.js +91 -0
- package/dist/src/sessions/workspace-discovery.js +83 -7
- package/dist/src/skills/adapters/local.js +32 -9
- package/dist/src/skills/agent-skill-sidecar.js +28 -0
- package/dist/src/skills/agent-skills.js +6 -23
- package/dist/src/skills/claude-surface-projection.js +25 -0
- package/dist/src/skills/cli-display.js +33 -0
- package/dist/src/skills/cli.js +2 -26
- package/dist/src/skills/collision-discovery.js +84 -0
- package/dist/src/skills/deployer.js +26 -361
- package/dist/src/skills/deployment-ownership.js +74 -0
- package/dist/src/skills/deployment-policy.js +114 -0
- package/dist/src/skills/deployment-projection.js +152 -0
- package/dist/src/skills/deployment-types.js +20 -0
- package/dist/src/skills/doctor.js +2 -2
- package/dist/src/skills/exporter.js +8 -31
- package/dist/src/skills/importer.js +4 -26
- package/dist/src/skills/provider-extensions.js +41 -0
- package/dist/src/skills/registry.js +22 -7
- package/dist/src/skills/run.js +6 -0
- package/dist/src/skills/types.js +0 -8
- package/dist/src/skills/validator-resources.js +68 -0
- package/dist/src/skills/validator.js +6 -55
- package/dist/src/smiths/context-pipeline/parallelism-section.js +7 -3
- package/dist/src/storage/cli-migrate-options.js +125 -0
- package/dist/src/storage/cli-migrate.js +111 -0
- package/dist/src/storage/cli.js +4 -279
- package/dist/src/storage/migration-adapter.js +79 -0
- package/dist/src/storage/migration-copy.js +108 -0
- package/dist/src/storage/migration-protocol.js +14 -197
- package/dist/src/storage/migration-records.js +186 -0
- package/dist/src/storage/migration-types.js +12 -0
- package/dist/src/tracker/forge-http.js +2 -1
- package/dist/src/tracker/issue-pages.js +110 -0
- package/package.json +3 -1
- package/schemas/mission-v1.schema.json +335 -0
- package/schemas/models/model-catalog.v1.schema.json +19 -1
- package/schemas/models/model-config-compatibility.v1.schema.json +3 -2
- package/schemas/models/model-policy.v1.schema.json +1 -1
- package/tools/agents/providers/antigravity.mjs +3 -5
- package/tools/agents/providers/base.mjs +3 -3
- package/tools/agents/providers/claude.mjs +3 -5
- package/tools/agents/providers/codex.mjs +3 -5
- package/tools/agents/providers/copilot.mjs +3 -5
- package/tools/agents/providers/cursor.mjs +3 -5
- package/tools/agents/providers/factory.mjs +3 -5
- package/tools/agents/providers/grok-build.mjs +9 -7
- package/tools/agents/providers/model-wrapper.mjs +86 -0
- package/tools/agents/providers/omp.mjs +3 -1
- package/tools/agents/providers/openclaw.mjs +3 -5
- package/tools/agents/providers/opencode.mjs +3 -5
- package/tools/agents/providers/provider-paths-staging.mjs +44 -0
- package/tools/agents/providers/rule-binding-targets.mjs +146 -0
- package/tools/agents/providers/rule-delivery.mjs +9 -128
- package/tools/agents/providers/warp.mjs +3 -5
- package/tools/agents/providers/windsurf.mjs +3 -5
- package/tools/plugin/package-plugins.mjs +2 -2
- package/tools/skills/codex-skill-transform.mjs +232 -0
- package/tools/skills/deploy-skills-codex.mjs +8 -249
- package/tools/warp/warp-content.mjs +548 -0
- package/vscode-extension/schemas/aiwg.config.v1.json +177 -0
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
// Parses migration flags and resolves existing storage adapters.
|
|
2
|
+
import { join, resolve as resolvePath } from 'node:path';
|
|
3
|
+
import { BACKEND_TYPES, SUBSYSTEM_KEYS, FilesystemAdapter, ObsidianAdapter, LogseqAdapter, FortemiAdapter, } from './index.js';
|
|
4
|
+
export function parseMigrateArgs(args) {
|
|
5
|
+
let subsystem;
|
|
6
|
+
let from;
|
|
7
|
+
let to;
|
|
8
|
+
let fromFolder;
|
|
9
|
+
let toFolder;
|
|
10
|
+
let dryRun = false;
|
|
11
|
+
let textOnly = false;
|
|
12
|
+
for (let i = 0; i < args.length; i++) {
|
|
13
|
+
const a = args[i];
|
|
14
|
+
if (a === '--from')
|
|
15
|
+
from = args[++i];
|
|
16
|
+
else if (a === '--to')
|
|
17
|
+
to = args[++i];
|
|
18
|
+
else if (a === '--from-folder')
|
|
19
|
+
fromFolder = args[++i];
|
|
20
|
+
else if (a === '--to-folder')
|
|
21
|
+
toFolder = args[++i];
|
|
22
|
+
else if (a === '--dry-run')
|
|
23
|
+
dryRun = true;
|
|
24
|
+
else if (a === '--text-only')
|
|
25
|
+
textOnly = true;
|
|
26
|
+
else if (!a.startsWith('--') && !subsystem)
|
|
27
|
+
subsystem = a;
|
|
28
|
+
else
|
|
29
|
+
throw new Error(`Unknown migrate flag: ${a}`);
|
|
30
|
+
}
|
|
31
|
+
if (!subsystem || !SUBSYSTEM_KEYS.includes(subsystem)) {
|
|
32
|
+
throw new Error(`Usage: aiwg storage migrate <subsystem> --from <type>:<location> --to <type>:<location> [--dry-run]\n Valid subsystems: ${SUBSYSTEM_KEYS.join(', ')}`);
|
|
33
|
+
}
|
|
34
|
+
if (!from)
|
|
35
|
+
throw new Error('storage migrate: --from <type>:<location> is required');
|
|
36
|
+
if (!to)
|
|
37
|
+
throw new Error('storage migrate: --to <type>:<location> is required');
|
|
38
|
+
return {
|
|
39
|
+
subsystem: subsystem,
|
|
40
|
+
from: { ...parseSpec(from), ...(fromFolder ? { folder: fromFolder } : {}) },
|
|
41
|
+
to: { ...parseSpec(to), ...(toFolder ? { folder: toFolder } : {}) },
|
|
42
|
+
dryRun,
|
|
43
|
+
textOnly,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
export function parseSpec(raw) {
|
|
47
|
+
const idx = raw.indexOf(':');
|
|
48
|
+
if (idx === -1) {
|
|
49
|
+
throw new Error(`Invalid backend spec "${raw}" — expected <type>:<location>`);
|
|
50
|
+
}
|
|
51
|
+
const type = raw.slice(0, idx);
|
|
52
|
+
const location = raw.slice(idx + 1);
|
|
53
|
+
if (!BACKEND_TYPES.includes(type)) {
|
|
54
|
+
throw new Error(`Unknown backend type "${type}" in spec "${raw}". Valid: ${BACKEND_TYPES.join(', ')}`);
|
|
55
|
+
}
|
|
56
|
+
if (location.length === 0) {
|
|
57
|
+
throw new Error(`Empty location in backend spec "${raw}"`);
|
|
58
|
+
}
|
|
59
|
+
return { type: type, location };
|
|
60
|
+
}
|
|
61
|
+
export function buildAdapter(subsystem, spec, projectRoot) {
|
|
62
|
+
switch (spec.type) {
|
|
63
|
+
case 'fs': {
|
|
64
|
+
const root = expandFsLocation(spec.location, projectRoot);
|
|
65
|
+
return new FilesystemAdapter(root);
|
|
66
|
+
}
|
|
67
|
+
case 'obsidian':
|
|
68
|
+
return new ObsidianAdapter({
|
|
69
|
+
type: 'obsidian',
|
|
70
|
+
vault: expandFsLocation(spec.location, projectRoot),
|
|
71
|
+
...(spec.folder ? { folder: spec.folder } : {}),
|
|
72
|
+
useCli: false,
|
|
73
|
+
});
|
|
74
|
+
case 'logseq':
|
|
75
|
+
return new LogseqAdapter({
|
|
76
|
+
type: 'logseq',
|
|
77
|
+
graph: expandFsLocation(spec.location, projectRoot),
|
|
78
|
+
useApi: false,
|
|
79
|
+
});
|
|
80
|
+
case 'fortemi':
|
|
81
|
+
return new FortemiAdapter({
|
|
82
|
+
subsystem,
|
|
83
|
+
config: { type: 'fortemi', mcpServer: spec.location },
|
|
84
|
+
});
|
|
85
|
+
case 'notion':
|
|
86
|
+
case 'anythingllm':
|
|
87
|
+
case 's3':
|
|
88
|
+
case 'webdav':
|
|
89
|
+
throw new Error(`storage migrate: backend "${spec.type}" not yet implemented. ` +
|
|
90
|
+
`See issues #959, #960, #962, #963 for tracking.`);
|
|
91
|
+
default: {
|
|
92
|
+
const _exhaustive = spec.type;
|
|
93
|
+
void _exhaustive;
|
|
94
|
+
throw new Error(`Unhandled backend type ${spec.type}`);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
export function expandFsLocation(loc, projectRoot) {
|
|
99
|
+
if (loc.startsWith('~/')) {
|
|
100
|
+
const { homedir } = require('os');
|
|
101
|
+
return join(homedir(), loc.slice(2));
|
|
102
|
+
}
|
|
103
|
+
if (loc === '~') {
|
|
104
|
+
const { homedir } = require('os');
|
|
105
|
+
return homedir();
|
|
106
|
+
}
|
|
107
|
+
if (loc.startsWith('/'))
|
|
108
|
+
return loc;
|
|
109
|
+
return resolvePath(projectRoot, loc);
|
|
110
|
+
}
|
|
111
|
+
export function locationsEqual(a, b, projectRoot) {
|
|
112
|
+
if (a.type !== b.type)
|
|
113
|
+
return false;
|
|
114
|
+
if (a.type === 'fortemi')
|
|
115
|
+
return a.location === b.location;
|
|
116
|
+
// For file-shaped backends, compare resolved fs paths
|
|
117
|
+
return expandFsLocation(a.location, projectRoot) === expandFsLocation(b.location, projectRoot);
|
|
118
|
+
}
|
|
119
|
+
export function describeSpec(spec) {
|
|
120
|
+
return spec.folder ? `${spec.type}:${spec.location} (folder=${spec.folder})` : `${spec.type}:${spec.location}`;
|
|
121
|
+
}
|
|
122
|
+
export function specSlug(spec) {
|
|
123
|
+
return `${spec.type}-${spec.location.replace(/[^a-zA-Z0-9]/g, '_').slice(0, 32)}`;
|
|
124
|
+
}
|
|
125
|
+
//# sourceMappingURL=cli-migrate-options.js.map
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
// Runs storage migrate through the protocol copy coordinator and preserves CLI output.
|
|
2
|
+
import { randomUUID } from 'node:crypto';
|
|
3
|
+
import { appendFile, mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
|
|
4
|
+
import { dirname } from 'node:path';
|
|
5
|
+
import { projectAiwgPath } from '../config/project-artifacts.js';
|
|
6
|
+
import { STORAGE_MIGRATION_PROTOCOL, StorageMigrationCoordinator } from './migration-protocol.js';
|
|
7
|
+
import { isTextCorpusEntry, migrationEndpoint } from './migration-adapter.js';
|
|
8
|
+
import { sha256, stableStringify } from './migration-records.js';
|
|
9
|
+
import { buildAdapter, describeSpec, expandFsLocation, locationsEqual, parseMigrateArgs, specSlug, } from './cli-migrate-options.js';
|
|
10
|
+
export async function handleMigrate(projectRoot, args) {
|
|
11
|
+
const opts = parseMigrateArgs(args);
|
|
12
|
+
if (locationsEqual(opts.from, opts.to, projectRoot)) {
|
|
13
|
+
throw new Error(`Refusing to migrate: source and destination resolve to the same location.\n ${describeSpec(opts.from)}\n ${describeSpec(opts.to)}`);
|
|
14
|
+
}
|
|
15
|
+
const source = buildAdapter(opts.subsystem, opts.from, projectRoot);
|
|
16
|
+
const destination = buildAdapter(opts.subsystem, opts.to, projectRoot);
|
|
17
|
+
const identity = (spec) => ({
|
|
18
|
+
backend: spec.type, instance: stableStringify({
|
|
19
|
+
location: spec.type === 'fortemi' ? spec.location : expandFsLocation(spec.location, projectRoot),
|
|
20
|
+
folder: spec.folder ?? '',
|
|
21
|
+
}), tenant: projectRoot, subsystem: opts.subsystem, schemaVersion: '1',
|
|
22
|
+
});
|
|
23
|
+
const sourceEndpoint = migrationEndpoint(source, identity(opts.from));
|
|
24
|
+
const destinationEndpoint = migrationEndpoint(destination, identity(opts.to));
|
|
25
|
+
const migrationLogPath = projectAiwgPath(projectRoot, '.storage-cache', 'migrations', `${opts.subsystem}-${specSlug(opts.from)}-to-${specSlug(opts.to)}.jsonl`);
|
|
26
|
+
// Bind checkpoint filenames to full, resolved identities, folders and selection policy.
|
|
27
|
+
const binding = sha256(stableStringify([sourceEndpoint.identity, destinationEndpoint.identity, opts.textOnly]));
|
|
28
|
+
const checkpointPath = `${migrationLogPath}.${binding}.checkpoint.json`;
|
|
29
|
+
const receiptPath = `${checkpointPath}.receipt.json`;
|
|
30
|
+
try {
|
|
31
|
+
if (source.init)
|
|
32
|
+
await source.init();
|
|
33
|
+
if (!opts.dryRun && destination.init)
|
|
34
|
+
await destination.init();
|
|
35
|
+
console.log(`storage migrate (${opts.dryRun ? 'DRY RUN' : 'live'})`);
|
|
36
|
+
console.log(` subsystem: ${opts.subsystem}`);
|
|
37
|
+
console.log(` from: ${describeSpec(opts.from)}`);
|
|
38
|
+
console.log(` to: ${describeSpec(opts.to)}`);
|
|
39
|
+
console.log('');
|
|
40
|
+
const checkpoint = await loadCheckpoint(checkpointPath);
|
|
41
|
+
const result = await StorageMigrationCoordinator.copy({
|
|
42
|
+
source: sourceEndpoint, destination: destinationEndpoint, checkpoint, dryRun: opts.dryRun,
|
|
43
|
+
exclude: path => opts.textOnly && !isTextCorpusEntry(path) ? 'non-text attachment skipped' : undefined,
|
|
44
|
+
async save(value) {
|
|
45
|
+
await atomicJson(checkpointPath, value);
|
|
46
|
+
await atomicJson(receiptPath, { protocol: value.protocol, migrationId: value.migrationId, ...value.receipt });
|
|
47
|
+
},
|
|
48
|
+
async report(action, item) {
|
|
49
|
+
if (action === 'errored')
|
|
50
|
+
console.log(` ✗ ${item.path} (${item.reason})`);
|
|
51
|
+
else if (action === 'skipped') {
|
|
52
|
+
const marker = item.reason === 'already migrated, skipped' ? '✓' : '·';
|
|
53
|
+
console.log(` ${marker} ${item.path} (${item.reason})`);
|
|
54
|
+
}
|
|
55
|
+
else if (opts.dryRun) {
|
|
56
|
+
console.log(` → ${item.path} (would ${action === 'deleted' ? 'delete' : 'copy'})`);
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
console.log(` ✓ ${item.path}${action === 'copied' ? '' : ` (${action}: ${item.reason})`}`);
|
|
60
|
+
await mkdir(dirname(migrationLogPath), { recursive: true });
|
|
61
|
+
await appendFile(migrationLogPath, JSON.stringify({
|
|
62
|
+
path: item.path, ts: new Date().toISOString(), action, reason: item.reason, protocol: STORAGE_MIGRATION_PROTOCOL,
|
|
63
|
+
}) + '\n', 'utf-8');
|
|
64
|
+
}
|
|
65
|
+
},
|
|
66
|
+
});
|
|
67
|
+
const receipt = result.receipt;
|
|
68
|
+
const unsupported = receipt.skipped.filter(item => item.reason === 'non-text attachment skipped').length;
|
|
69
|
+
const copied = receipt.copied.length + receipt.recopied.length;
|
|
70
|
+
const skipped = receipt.skipped.length - unsupported;
|
|
71
|
+
if (!result.plan.length)
|
|
72
|
+
console.log('No entries to migrate.');
|
|
73
|
+
console.log('');
|
|
74
|
+
console.log(`Summary: copied=${copied} skipped=${skipped} unsupported=${unsupported} ` +
|
|
75
|
+
`errored=${receipt.errored.length} total=${result.plan.length} ` +
|
|
76
|
+
`recopied=${receipt.recopied.length} deleted=${receipt.deleted.length}`);
|
|
77
|
+
if (!opts.dryRun) {
|
|
78
|
+
console.log(`Migration log: ${migrationLogPath}`);
|
|
79
|
+
console.log(`Migration checkpoint: ${checkpointPath}`);
|
|
80
|
+
console.log(`Migration receipt: ${receiptPath}`);
|
|
81
|
+
}
|
|
82
|
+
if (receipt.errored.length) {
|
|
83
|
+
throw new Error(`${receipt.errored.length} entr${receipt.errored.length === 1 ? 'y' : 'ies'} failed to migrate.`);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
finally {
|
|
87
|
+
await Promise.allSettled([source.close?.(), destination.close?.()]);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
async function loadCheckpoint(path) {
|
|
91
|
+
try {
|
|
92
|
+
return JSON.parse(await readFile(path, 'utf-8'));
|
|
93
|
+
}
|
|
94
|
+
catch (error) {
|
|
95
|
+
if (error.code === 'ENOENT')
|
|
96
|
+
return undefined;
|
|
97
|
+
throw error;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
async function atomicJson(path, value) {
|
|
101
|
+
await mkdir(dirname(path), { recursive: true });
|
|
102
|
+
const temporary = `${path}.${randomUUID()}.tmp`;
|
|
103
|
+
try {
|
|
104
|
+
await writeFile(temporary, JSON.stringify(value, null, 2) + '\n', 'utf-8');
|
|
105
|
+
await rename(temporary, path);
|
|
106
|
+
}
|
|
107
|
+
finally {
|
|
108
|
+
await rm(temporary, { force: true });
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
//# sourceMappingURL=cli-migrate.js.map
|
package/dist/src/storage/cli.js
CHANGED
|
@@ -15,12 +15,10 @@
|
|
|
15
15
|
* @issue #955
|
|
16
16
|
*/
|
|
17
17
|
import { randomUUID } from 'crypto';
|
|
18
|
-
import {
|
|
19
|
-
|
|
20
|
-
import {
|
|
21
|
-
import {
|
|
22
|
-
import { BACKEND_TYPES, FilesystemAdapter, ObsidianAdapter, LogseqAdapter, FortemiAdapter, SUBSYSTEM_KEYS, getLoadedConfig, initStorage, resolveStorage, resolveSubsystemRoot, storageConfigPath, } from './index.js';
|
|
23
|
-
import { projectAiwgPath, resolveProjectAiwgDir } from '../config/project-artifacts.js';
|
|
18
|
+
import { handleMigrate } from './cli-migrate.js';
|
|
19
|
+
export { isTextCorpusEntry } from './migration-adapter.js';
|
|
20
|
+
import { BACKEND_TYPES, SUBSYSTEM_KEYS, getLoadedConfig, initStorage, resolveStorage, resolveSubsystemRoot, storageConfigPath, } from './index.js';
|
|
21
|
+
import { resolveProjectAiwgDir } from '../config/project-artifacts.js';
|
|
24
22
|
export async function main(args, projectRoot = process.cwd()) {
|
|
25
23
|
const subcommand = args[0];
|
|
26
24
|
const subArgs = args.slice(1);
|
|
@@ -141,166 +139,6 @@ async function handleTest(projectRoot, args) {
|
|
|
141
139
|
` Subsystem: ${subsystem}\n Probe path: ${probePath}`);
|
|
142
140
|
}
|
|
143
141
|
}
|
|
144
|
-
/**
|
|
145
|
-
* `aiwg storage migrate <subsystem> --from <spec> --to <spec> [--dry-run]`
|
|
146
|
-
*
|
|
147
|
-
* Spec format: `<type>:<location>` where type is one of fs/obsidian/
|
|
148
|
-
* logseq/fortemi and location is:
|
|
149
|
-
* fs → directory path (relative to projectRoot or absolute)
|
|
150
|
-
* obsidian → vault directory (folder via `--from-folder`/`--to-folder`)
|
|
151
|
-
* logseq → graph directory
|
|
152
|
-
* fortemi → MCP server name (default: 'fortemi')
|
|
153
|
-
*
|
|
154
|
-
* Examples:
|
|
155
|
-
* aiwg storage migrate memory --from fs:.aiwg/memory --to obsidian:~/vault --to-folder AIWG/memory
|
|
156
|
-
* aiwg storage migrate kb --from fs:.aiwg/kb --to fortemi:fortemi --dry-run
|
|
157
|
-
*
|
|
158
|
-
* Walks source.list(''), reads each entry, writes to destination. Tracks
|
|
159
|
-
* progress in `.aiwg/.storage-cache/migrations/<id>.jsonl` so re-running
|
|
160
|
-
* resumes from the last successfully-migrated entry.
|
|
161
|
-
*
|
|
162
|
-
* @issue #955
|
|
163
|
-
*/
|
|
164
|
-
async function handleMigrate(projectRoot, args) {
|
|
165
|
-
const opts = parseMigrateArgs(args);
|
|
166
|
-
const source = buildAdapter(opts.subsystem, opts.from, projectRoot);
|
|
167
|
-
const destination = buildAdapter(opts.subsystem, opts.to, projectRoot);
|
|
168
|
-
if (locationsEqual(opts.from, opts.to, projectRoot)) {
|
|
169
|
-
throw new Error(`Refusing to migrate: source and destination resolve to the same location.\n ${describeSpec(opts.from)}\n ${describeSpec(opts.to)}`);
|
|
170
|
-
}
|
|
171
|
-
if (source.init)
|
|
172
|
-
await source.init();
|
|
173
|
-
if (!opts.dryRun && destination.init)
|
|
174
|
-
await destination.init();
|
|
175
|
-
console.log(`storage migrate (${opts.dryRun ? 'DRY RUN' : 'live'})`);
|
|
176
|
-
console.log(` subsystem: ${opts.subsystem}`);
|
|
177
|
-
console.log(` from: ${describeSpec(opts.from)}`);
|
|
178
|
-
console.log(` to: ${describeSpec(opts.to)}`);
|
|
179
|
-
console.log('');
|
|
180
|
-
const entries = await source.list('');
|
|
181
|
-
if (entries.length === 0) {
|
|
182
|
-
console.log('No entries to migrate.');
|
|
183
|
-
return;
|
|
184
|
-
}
|
|
185
|
-
// Resume support: read the per-migration log; skip already-done paths
|
|
186
|
-
const migrationLogPath = projectAiwgPath(projectRoot, '.storage-cache', 'migrations', `${opts.subsystem}-${specSlug(opts.from)}-to-${specSlug(opts.to)}.jsonl`);
|
|
187
|
-
const completed = await readCompletedSet(migrationLogPath);
|
|
188
|
-
let copied = 0;
|
|
189
|
-
let skipped = 0;
|
|
190
|
-
let errored = 0;
|
|
191
|
-
let unsupported = 0;
|
|
192
|
-
for (const entry of entries) {
|
|
193
|
-
if (opts.textOnly && !isTextCorpusEntry(entry.path)) {
|
|
194
|
-
unsupported++;
|
|
195
|
-
console.log(` · ${entry.path} (non-text attachment skipped)`);
|
|
196
|
-
continue;
|
|
197
|
-
}
|
|
198
|
-
if (completed.has(entry.path)) {
|
|
199
|
-
skipped++;
|
|
200
|
-
console.log(` ✓ ${entry.path} (already migrated, skipped)`);
|
|
201
|
-
continue;
|
|
202
|
-
}
|
|
203
|
-
if (opts.dryRun) {
|
|
204
|
-
copied++;
|
|
205
|
-
console.log(` → ${entry.path} (would copy)`);
|
|
206
|
-
continue;
|
|
207
|
-
}
|
|
208
|
-
try {
|
|
209
|
-
const content = await source.read(entry.path);
|
|
210
|
-
if (content === null) {
|
|
211
|
-
errored++;
|
|
212
|
-
console.log(` ✗ ${entry.path} (read returned null)`);
|
|
213
|
-
continue;
|
|
214
|
-
}
|
|
215
|
-
await destination.write(entry.path, content, migrationMetadata(entry.path, content));
|
|
216
|
-
await recordCompletion(migrationLogPath, entry.path);
|
|
217
|
-
copied++;
|
|
218
|
-
console.log(` ✓ ${entry.path}`);
|
|
219
|
-
}
|
|
220
|
-
catch (err) {
|
|
221
|
-
errored++;
|
|
222
|
-
console.log(` ✗ ${entry.path} (${err.message})`);
|
|
223
|
-
}
|
|
224
|
-
}
|
|
225
|
-
if (source.close)
|
|
226
|
-
await source.close();
|
|
227
|
-
if (destination.close)
|
|
228
|
-
await destination.close();
|
|
229
|
-
console.log('');
|
|
230
|
-
console.log(`Summary: copied=${copied} skipped=${skipped} unsupported=${unsupported} ` +
|
|
231
|
-
`errored=${errored} total=${entries.length}`);
|
|
232
|
-
if (!opts.dryRun) {
|
|
233
|
-
console.log(`Migration log: ${migrationLogPath}`);
|
|
234
|
-
}
|
|
235
|
-
if (errored > 0) {
|
|
236
|
-
throw new Error(`${errored} entr${errored === 1 ? 'y' : 'ies'} failed to migrate.`);
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
function parseMigrateArgs(args) {
|
|
240
|
-
let subsystem;
|
|
241
|
-
let from;
|
|
242
|
-
let to;
|
|
243
|
-
let fromFolder;
|
|
244
|
-
let toFolder;
|
|
245
|
-
let dryRun = false;
|
|
246
|
-
let textOnly = false;
|
|
247
|
-
for (let i = 0; i < args.length; i++) {
|
|
248
|
-
const a = args[i];
|
|
249
|
-
if (a === '--from')
|
|
250
|
-
from = args[++i];
|
|
251
|
-
else if (a === '--to')
|
|
252
|
-
to = args[++i];
|
|
253
|
-
else if (a === '--from-folder')
|
|
254
|
-
fromFolder = args[++i];
|
|
255
|
-
else if (a === '--to-folder')
|
|
256
|
-
toFolder = args[++i];
|
|
257
|
-
else if (a === '--dry-run')
|
|
258
|
-
dryRun = true;
|
|
259
|
-
else if (a === '--text-only')
|
|
260
|
-
textOnly = true;
|
|
261
|
-
else if (!a.startsWith('--') && !subsystem)
|
|
262
|
-
subsystem = a;
|
|
263
|
-
else
|
|
264
|
-
throw new Error(`Unknown migrate flag: ${a}`);
|
|
265
|
-
}
|
|
266
|
-
if (!subsystem || !SUBSYSTEM_KEYS.includes(subsystem)) {
|
|
267
|
-
throw new Error(`Usage: aiwg storage migrate <subsystem> --from <type>:<location> --to <type>:<location> [--dry-run]\n Valid subsystems: ${SUBSYSTEM_KEYS.join(', ')}`);
|
|
268
|
-
}
|
|
269
|
-
if (!from)
|
|
270
|
-
throw new Error('storage migrate: --from <type>:<location> is required');
|
|
271
|
-
if (!to)
|
|
272
|
-
throw new Error('storage migrate: --to <type>:<location> is required');
|
|
273
|
-
return {
|
|
274
|
-
subsystem: subsystem,
|
|
275
|
-
from: { ...parseSpec(from), ...(fromFolder ? { folder: fromFolder } : {}) },
|
|
276
|
-
to: { ...parseSpec(to), ...(toFolder ? { folder: toFolder } : {}) },
|
|
277
|
-
dryRun,
|
|
278
|
-
textOnly,
|
|
279
|
-
};
|
|
280
|
-
}
|
|
281
|
-
const TEXT_CORPUS_EXTENSIONS = new Set([
|
|
282
|
-
'.bib',
|
|
283
|
-
'.csv',
|
|
284
|
-
'.htm',
|
|
285
|
-
'.html',
|
|
286
|
-
'.json',
|
|
287
|
-
'.md',
|
|
288
|
-
'.ris',
|
|
289
|
-
'.txt',
|
|
290
|
-
'.xml',
|
|
291
|
-
'.yaml',
|
|
292
|
-
'.yml',
|
|
293
|
-
]);
|
|
294
|
-
export function isTextCorpusEntry(entryPath) {
|
|
295
|
-
return TEXT_CORPUS_EXTENSIONS.has(extname(entryPath).toLowerCase());
|
|
296
|
-
}
|
|
297
|
-
function migrationMetadata(entryPath, content) {
|
|
298
|
-
const extension = extname(entryPath).toLowerCase();
|
|
299
|
-
const contentType = extension === '.md' ? 'text/markdown' : 'text/plain';
|
|
300
|
-
if (extension !== '.md')
|
|
301
|
-
return { contentType };
|
|
302
|
-
return { contentType, frontmatter: parseFrontmatter(content).data };
|
|
303
|
-
}
|
|
304
142
|
async function handleImportCorpus(projectRoot, args) {
|
|
305
143
|
let server = 'fortemi';
|
|
306
144
|
let destination;
|
|
@@ -342,119 +180,6 @@ async function handleImportCorpus(projectRoot, args) {
|
|
|
342
180
|
...(dryRun ? ['--dry-run'] : []),
|
|
343
181
|
]);
|
|
344
182
|
}
|
|
345
|
-
function parseSpec(raw) {
|
|
346
|
-
const idx = raw.indexOf(':');
|
|
347
|
-
if (idx === -1) {
|
|
348
|
-
throw new Error(`Invalid backend spec "${raw}" — expected <type>:<location>`);
|
|
349
|
-
}
|
|
350
|
-
const type = raw.slice(0, idx);
|
|
351
|
-
const location = raw.slice(idx + 1);
|
|
352
|
-
if (!BACKEND_TYPES.includes(type)) {
|
|
353
|
-
throw new Error(`Unknown backend type "${type}" in spec "${raw}". Valid: ${BACKEND_TYPES.join(', ')}`);
|
|
354
|
-
}
|
|
355
|
-
if (location.length === 0) {
|
|
356
|
-
throw new Error(`Empty location in backend spec "${raw}"`);
|
|
357
|
-
}
|
|
358
|
-
return { type: type, location };
|
|
359
|
-
}
|
|
360
|
-
function buildAdapter(subsystem, spec, projectRoot) {
|
|
361
|
-
switch (spec.type) {
|
|
362
|
-
case 'fs': {
|
|
363
|
-
const root = expandFsLocation(spec.location, projectRoot);
|
|
364
|
-
return new FilesystemAdapter(root);
|
|
365
|
-
}
|
|
366
|
-
case 'obsidian':
|
|
367
|
-
return new ObsidianAdapter({
|
|
368
|
-
type: 'obsidian',
|
|
369
|
-
vault: expandFsLocation(spec.location, projectRoot),
|
|
370
|
-
...(spec.folder ? { folder: spec.folder } : {}),
|
|
371
|
-
useCli: false,
|
|
372
|
-
});
|
|
373
|
-
case 'logseq':
|
|
374
|
-
return new LogseqAdapter({
|
|
375
|
-
type: 'logseq',
|
|
376
|
-
graph: expandFsLocation(spec.location, projectRoot),
|
|
377
|
-
useApi: false,
|
|
378
|
-
});
|
|
379
|
-
case 'fortemi':
|
|
380
|
-
return new FortemiAdapter({
|
|
381
|
-
subsystem,
|
|
382
|
-
config: { type: 'fortemi', mcpServer: spec.location },
|
|
383
|
-
});
|
|
384
|
-
case 'notion':
|
|
385
|
-
case 'anythingllm':
|
|
386
|
-
case 's3':
|
|
387
|
-
case 'webdav':
|
|
388
|
-
throw new Error(`storage migrate: backend "${spec.type}" not yet implemented. ` +
|
|
389
|
-
`See issues #959, #960, #962, #963 for tracking.`);
|
|
390
|
-
default: {
|
|
391
|
-
const _exhaustive = spec.type;
|
|
392
|
-
void _exhaustive;
|
|
393
|
-
throw new Error(`Unhandled backend type ${spec.type}`);
|
|
394
|
-
}
|
|
395
|
-
}
|
|
396
|
-
}
|
|
397
|
-
function expandFsLocation(loc, projectRoot) {
|
|
398
|
-
if (loc.startsWith('~/')) {
|
|
399
|
-
const { homedir } = require('os');
|
|
400
|
-
return join(homedir(), loc.slice(2));
|
|
401
|
-
}
|
|
402
|
-
if (loc === '~') {
|
|
403
|
-
const { homedir } = require('os');
|
|
404
|
-
return homedir();
|
|
405
|
-
}
|
|
406
|
-
if (loc.startsWith('/'))
|
|
407
|
-
return loc;
|
|
408
|
-
return resolvePath(projectRoot, loc);
|
|
409
|
-
}
|
|
410
|
-
function locationsEqual(a, b, projectRoot) {
|
|
411
|
-
if (a.type !== b.type)
|
|
412
|
-
return false;
|
|
413
|
-
if (a.type === 'fortemi')
|
|
414
|
-
return a.location === b.location;
|
|
415
|
-
// For file-shaped backends, compare resolved fs paths
|
|
416
|
-
return expandFsLocation(a.location, projectRoot) === expandFsLocation(b.location, projectRoot);
|
|
417
|
-
}
|
|
418
|
-
function describeSpec(spec) {
|
|
419
|
-
return spec.folder ? `${spec.type}:${spec.location} (folder=${spec.folder})` : `${spec.type}:${spec.location}`;
|
|
420
|
-
}
|
|
421
|
-
function specSlug(spec) {
|
|
422
|
-
return `${spec.type}-${spec.location.replace(/[^a-zA-Z0-9]/g, '_').slice(0, 32)}`;
|
|
423
|
-
}
|
|
424
|
-
async function readCompletedSet(logPath) {
|
|
425
|
-
if (!existsSync(logPath))
|
|
426
|
-
return new Set();
|
|
427
|
-
try {
|
|
428
|
-
const content = await readFile(logPath, 'utf-8');
|
|
429
|
-
const out = new Set();
|
|
430
|
-
for (const line of content.split('\n')) {
|
|
431
|
-
if (!line.trim())
|
|
432
|
-
continue;
|
|
433
|
-
try {
|
|
434
|
-
const obj = JSON.parse(line);
|
|
435
|
-
if (obj.path)
|
|
436
|
-
out.add(obj.path);
|
|
437
|
-
}
|
|
438
|
-
catch {
|
|
439
|
-
/* ignore malformed line */
|
|
440
|
-
}
|
|
441
|
-
}
|
|
442
|
-
return out;
|
|
443
|
-
}
|
|
444
|
-
catch {
|
|
445
|
-
return new Set();
|
|
446
|
-
}
|
|
447
|
-
}
|
|
448
|
-
async function recordCompletion(logPath, path) {
|
|
449
|
-
await mkdir(dirname(logPath), { recursive: true });
|
|
450
|
-
const line = JSON.stringify({ path, ts: new Date().toISOString() }) + '\n';
|
|
451
|
-
if (existsSync(logPath)) {
|
|
452
|
-
await appendFile(logPath, line, 'utf-8');
|
|
453
|
-
}
|
|
454
|
-
else {
|
|
455
|
-
await writeFile(logPath, line, 'utf-8');
|
|
456
|
-
}
|
|
457
|
-
}
|
|
458
183
|
function describeBackendLocation(subsystem, config) {
|
|
459
184
|
const b = config?.backends?.[subsystem];
|
|
460
185
|
if (!b)
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
// Bridges text storage adapters to protocol records and single-record commits.
|
|
2
|
+
import { extname } from 'node:path';
|
|
3
|
+
import { randomUUID } from 'node:crypto';
|
|
4
|
+
import { parseFrontmatter } from '../artifacts/index-builder.js';
|
|
5
|
+
import { digestRecords, sha256 } from './migration-records.js';
|
|
6
|
+
export function migrationEndpoint(adapter, identity) {
|
|
7
|
+
const readRecord = async (path) => {
|
|
8
|
+
const value = await adapter.read(path);
|
|
9
|
+
if (value === null)
|
|
10
|
+
return null;
|
|
11
|
+
const digest = sha256(value);
|
|
12
|
+
return { identity: { tenant: identity.tenant, subsystem: identity.subsystem, path }, value, digest, sourceRevision: digest };
|
|
13
|
+
};
|
|
14
|
+
const readAll = async () => {
|
|
15
|
+
const records = [];
|
|
16
|
+
for (const { path } of await adapter.list('')) {
|
|
17
|
+
const record = await readRecord(path);
|
|
18
|
+
if (record === null)
|
|
19
|
+
throw new Error(`source changed during enumeration: ${path} (read returned null)`);
|
|
20
|
+
records.push(record);
|
|
21
|
+
}
|
|
22
|
+
return records;
|
|
23
|
+
};
|
|
24
|
+
return {
|
|
25
|
+
identity, readRecord, readAll,
|
|
26
|
+
async snapshot() {
|
|
27
|
+
const records = await readAll();
|
|
28
|
+
return { id: randomUUID(), highWaterMark: digestRecords(records), records };
|
|
29
|
+
},
|
|
30
|
+
async commitBatch(mutations) {
|
|
31
|
+
// Baseline adapters provide single-path idempotent writes, not multi-path transactions.
|
|
32
|
+
if (mutations.length !== 1)
|
|
33
|
+
throw new Error('storage adapter copy requires single-record batches');
|
|
34
|
+
const mutation = mutations[0];
|
|
35
|
+
const path = mutation.record.identity.path;
|
|
36
|
+
if ((await readRecord(path))?.sourceRevision !== mutation.expectedRevision) {
|
|
37
|
+
throw new Error(`destination changed before commit: ${path}`);
|
|
38
|
+
}
|
|
39
|
+
if (mutation.operation === 'delete')
|
|
40
|
+
await adapter.delete(path);
|
|
41
|
+
else
|
|
42
|
+
await adapter.write(path, mutation.record.value, migrationMetadata(path, mutation.record.value));
|
|
43
|
+
const committed = await readRecord(path);
|
|
44
|
+
if (mutation.operation === 'delete' && committed !== null)
|
|
45
|
+
throw new Error(`delete not applied: ${path}`);
|
|
46
|
+
if (mutation.operation === 'upsert' && committed === null)
|
|
47
|
+
throw new Error(`write not applied: ${path}`);
|
|
48
|
+
return {
|
|
49
|
+
batchId: randomUUID(), committed: true, highWaterMark: mutation.record.sourceRevision,
|
|
50
|
+
recordReceipts: [{ identity: mutation.record.identity, digest: mutation.record.digest,
|
|
51
|
+
sourceRevision: committed?.sourceRevision ?? mutation.record.sourceRevision }],
|
|
52
|
+
};
|
|
53
|
+
},
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
const TEXT_CORPUS_EXTENSIONS = new Set([
|
|
57
|
+
'.bib',
|
|
58
|
+
'.csv',
|
|
59
|
+
'.htm',
|
|
60
|
+
'.html',
|
|
61
|
+
'.json',
|
|
62
|
+
'.md',
|
|
63
|
+
'.ris',
|
|
64
|
+
'.txt',
|
|
65
|
+
'.xml',
|
|
66
|
+
'.yaml',
|
|
67
|
+
'.yml',
|
|
68
|
+
]);
|
|
69
|
+
export function isTextCorpusEntry(entryPath) {
|
|
70
|
+
return TEXT_CORPUS_EXTENSIONS.has(extname(entryPath).toLowerCase());
|
|
71
|
+
}
|
|
72
|
+
export function migrationMetadata(entryPath, content) {
|
|
73
|
+
const extension = extname(entryPath).toLowerCase();
|
|
74
|
+
const contentType = extension === '.md' ? 'text/markdown' : 'text/plain';
|
|
75
|
+
if (extension !== '.md')
|
|
76
|
+
return { contentType };
|
|
77
|
+
return { contentType, frontmatter: parseFrontmatter(content).data };
|
|
78
|
+
}
|
|
79
|
+
//# sourceMappingURL=migration-adapter.js.map
|