moflo 4.12.11 → 4.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/guidance/shipped/moflo-cli-reference.md +45 -1
- package/.claude/guidance/shipped/moflo-cross-install-memory-sharing.md +7 -2
- package/.claude/guidance/shipped/moflo-skills-reference.md +2 -0
- package/.claude/skills/fl/phases.md +51 -17
- package/.claude/skills/optimize-learnings/SKILL.md +220 -0
- package/README.md +95 -1
- package/bin/lib/get-backend.mjs +150 -12
- package/bin/lib/skill-categories.mjs +1 -0
- package/bin/session-start-launcher.mjs +13 -5
- package/dist/src/cli/commands/daemon.js +5 -2
- package/dist/src/cli/commands/epic.js +5 -1
- package/dist/src/cli/commands/hive-mind.js +6 -4
- package/dist/src/cli/commands/hooks.js +8 -8
- package/dist/src/cli/commands/index.js +5 -0
- package/dist/src/cli/commands/memory-audit-learnings.js +587 -0
- package/dist/src/cli/commands/memory.js +71 -10
- package/dist/src/cli/commands/spell-schedule.js +5 -3
- package/dist/src/cli/commands/worktree.js +408 -0
- package/dist/src/cli/config/moflo-config.js +57 -0
- package/dist/src/cli/index.js +4 -2
- package/dist/src/cli/init/executor.js +1 -0
- package/dist/src/cli/mcp-tools/memory-admin-tools.js +46 -8
- package/dist/src/cli/mcp-tools/moflodb-tools.js +30 -6
- package/dist/src/cli/memory/bridge-entries.js +157 -9
- package/dist/src/cli/memory/controllers/batch-operations.js +7 -2
- package/dist/src/cli/memory/daemon-backend.js +152 -11
- package/dist/src/cli/memory/entries-read.js +47 -2
- package/dist/src/cli/memory/entries-write.js +73 -10
- package/dist/src/cli/memory/hnsw-singleton.js +112 -9
- package/dist/src/cli/memory/learnings-audit.js +420 -0
- package/dist/src/cli/memory/learnings-dead-paths.js +202 -0
- package/dist/src/cli/memory/learnings-tree.js +187 -0
- package/dist/src/cli/memory/memory-bridge.js +37 -27
- package/dist/src/cli/memory/tool-call-markup.js +218 -0
- package/dist/src/cli/parser.js +7 -3
- package/dist/src/cli/services/cherry-pick-learnings.js +9 -3
- package/dist/src/cli/services/durable-reconcile.js +161 -0
- package/dist/src/cli/services/durable-store-io.js +291 -0
- package/dist/src/cli/services/durable-sync.js +159 -24
- package/dist/src/cli/services/team-artifact-sync.js +462 -163
- package/dist/src/cli/services/worktree-provision.js +400 -0
- package/dist/src/cli/version.js +1 -1
- package/package.json +2 -2
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Git-tracked team learnings artifact (#1234, epic #1231).
|
|
2
|
+
* Git-tracked team learnings artifact (#1234, epic #1231; reconciling since #1463).
|
|
3
3
|
*
|
|
4
4
|
* Story 1 (#1232) shared a durable SQLite store between worktrees; Story 2
|
|
5
5
|
* (#1233) carried a durable SQLite artifact between one user's machines. This
|
|
@@ -17,15 +17,41 @@
|
|
|
17
17
|
* normal index pass (or `flo memory rebuild-index`), exactly as any other
|
|
18
18
|
* unembedded row.
|
|
19
19
|
*
|
|
20
|
-
* Conflict policy:
|
|
21
|
-
* `UNIQUE(namespace, key)` constraint:
|
|
22
|
-
* - on re-export, an entry already in the artifact keeps its original line
|
|
23
|
-
* (and original author/timestamp) — we never rewrite a teammate's row.
|
|
24
|
-
* - on import, `INSERT OR IGNORE` means a row already present locally wins,
|
|
25
|
-
* and within the file the first line for a key wins.
|
|
20
|
+
* ## Conflict policy: last-writer-wins on `updated_at` (#1463)
|
|
26
21
|
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
22
|
+
* This **replaces** the original first-write-wins policy, which was the #1463
|
|
23
|
+
* bug rather than a design: export skipped any key already in the artifact and
|
|
24
|
+
* import ran `INSERT OR IGNORE`, so a correction and a deletion were silently
|
|
25
|
+
* dropped in both directions. The artifact is the effective source of truth
|
|
26
|
+
* across machines, which made the two operations that most need to propagate
|
|
27
|
+
* the two the pair could not do.
|
|
28
|
+
*
|
|
29
|
+
* Both directions now run the same {@link planReconcile} rule with source and
|
|
30
|
+
* target swapped. Ties change nothing, and a key present only in the target is
|
|
31
|
+
* never touched — that is what protects work authored locally and not yet
|
|
32
|
+
* shared. See `durable-reconcile.ts` for the full matrix.
|
|
33
|
+
*
|
|
34
|
+
* ## Tombstones
|
|
35
|
+
*
|
|
36
|
+
* A deletion travels as its own line, marked with a namespace that is not
|
|
37
|
+
* durable:
|
|
38
|
+
*
|
|
39
|
+
* ```json
|
|
40
|
+
* {"namespace":"__moflo_tombstone__","key":"purged-key",
|
|
41
|
+
* "deleted":{"namespace":"learnings","key":"purged-key","at":1787751507635},
|
|
42
|
+
* "provenance":{...}}
|
|
43
|
+
* ```
|
|
44
|
+
*
|
|
45
|
+
* That marker buys backward compatibility for free. `importTeamArtifact` has
|
|
46
|
+
* always routed non-durable namespaces to `skippedNonDurable`, so 4.12.11 and
|
|
47
|
+
* every earlier version silently ignore tombstones — they do not act on
|
|
48
|
+
* deletions, which is exactly their behaviour today. An old client's *export*
|
|
49
|
+
* also preserves the lines verbatim, since it re-serialises whatever it read.
|
|
50
|
+
* No version gate and no artifact-schema field are required.
|
|
51
|
+
*
|
|
52
|
+
* Provenance now records who wrote a line **last** rather than first, because a
|
|
53
|
+
* line can now be rewritten. That is the more useful fact when reviewing a diff
|
|
54
|
+
* that changes an entry.
|
|
29
55
|
*
|
|
30
56
|
* Opt-in: inert unless `memory.team_artifact` (or `MOFLO_TEAM_ARTIFACT`) points
|
|
31
57
|
* at a path. Solo users see byte-identical behaviour to today.
|
|
@@ -39,11 +65,13 @@ import { spawnSync } from 'child_process';
|
|
|
39
65
|
import { createHash } from 'crypto';
|
|
40
66
|
import { findProjectRoot } from './project-root.js';
|
|
41
67
|
import { memoryDbPath } from './moflo-paths.js';
|
|
42
|
-
import { MEMORY_SCHEMA_V3 } from '../memory/memory-initializer.js';
|
|
43
68
|
import { openDaemonDatabase } from '../memory/daemon-backend.js';
|
|
44
69
|
import { atomicWriteFileSync } from '../shared/utils/atomic-file-write.js';
|
|
45
70
|
import { loadMofloConfig } from '../config/moflo-config.js';
|
|
46
|
-
import {
|
|
71
|
+
import { isDurableNamespace } from './cherry-pick-learnings.js';
|
|
72
|
+
import { planReconcile, reconcileId, splitReconcileId, isPrunableTombstone, recordStamp, TOMBSTONE_TTL_MS, } from './durable-reconcile.js';
|
|
73
|
+
import { readDurableSnapshot, applyDurableActions, } from './durable-store-io.js';
|
|
74
|
+
import { detectToolCallMarkup } from '../memory/tool-call-markup.js';
|
|
47
75
|
/** Allowed `type` values — the schema CHECK set. An out-of-set value would make
|
|
48
76
|
* INSERT OR IGNORE silently drop a hand-edited artifact row, so we coerce. */
|
|
49
77
|
const VALID_TYPES = new Set([
|
|
@@ -56,8 +84,18 @@ const VALID_TYPES = new Set([
|
|
|
56
84
|
const coerceType = (t) => (t && VALID_TYPES.has(t) ? t : 'semantic');
|
|
57
85
|
/** Default artifact path, relative to the project root. POSIX-joined parts. */
|
|
58
86
|
export const DEFAULT_TEAM_ARTIFACT_REL = path.join('.moflo', 'shared', 'learnings.jsonl');
|
|
59
|
-
|
|
60
|
-
|
|
87
|
+
/**
|
|
88
|
+
* Namespace marker on a tombstone line. MUST stay outside
|
|
89
|
+
* {@link isDurableNamespace} — that is the entire backward-compatibility
|
|
90
|
+
* mechanism (old clients skip it as non-durable). Renaming it silently breaks
|
|
91
|
+
* deletion propagation for every client that hasn't upgraded, which is why
|
|
92
|
+
* `imports a tombstone-bearing artifact the way 4.12.11 does` pins it.
|
|
93
|
+
*/
|
|
94
|
+
export const TOMBSTONE_NAMESPACE = '__moflo_tombstone__';
|
|
95
|
+
/** True when a parsed line is a tombstone rather than a live entry. */
|
|
96
|
+
export function isTombstoneLine(line) {
|
|
97
|
+
return line.namespace === TOMBSTONE_NAMESPACE;
|
|
98
|
+
}
|
|
61
99
|
/**
|
|
62
100
|
* Resolve the configured team-artifact path to an absolute path, or `null` when
|
|
63
101
|
* the feature is off (no env, no `memory.team_artifact`). Precedence:
|
|
@@ -106,123 +144,314 @@ function resolveSource() {
|
|
|
106
144
|
return 'unknown-host';
|
|
107
145
|
}
|
|
108
146
|
}
|
|
109
|
-
/**
|
|
147
|
+
/** The (namespace, key) a line is *about* — its inner target for a tombstone. */
|
|
148
|
+
function lineId(line) {
|
|
149
|
+
return isTombstoneLine(line)
|
|
150
|
+
? reconcileId(line.deleted.namespace, line.deleted.key)
|
|
151
|
+
: reconcileId(line.namespace, line.key);
|
|
152
|
+
}
|
|
153
|
+
/** A parsed line as the merge rule sees it. See {@link TeamArtifactEntry.updated_at}. */
|
|
154
|
+
function lineToRecord(line) {
|
|
155
|
+
if (isTombstoneLine(line)) {
|
|
156
|
+
return {
|
|
157
|
+
namespace: line.deleted.namespace,
|
|
158
|
+
key: line.deleted.key,
|
|
159
|
+
updatedAt: 0,
|
|
160
|
+
deletedAt: line.deleted.at,
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
return {
|
|
164
|
+
namespace: line.namespace,
|
|
165
|
+
key: line.key,
|
|
166
|
+
updatedAt: typeof line.updated_at === 'number' ? line.updated_at : 0,
|
|
167
|
+
content: line.content ?? '',
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
/** Parse one JSONL line, or `null` when it is malformed. */
|
|
171
|
+
function parseLine(trimmed) {
|
|
172
|
+
let obj = null;
|
|
173
|
+
try {
|
|
174
|
+
obj = JSON.parse(trimmed);
|
|
175
|
+
}
|
|
176
|
+
catch {
|
|
177
|
+
return null;
|
|
178
|
+
}
|
|
179
|
+
if (!obj || typeof obj.namespace !== 'string' || typeof obj.key !== 'string')
|
|
180
|
+
return null;
|
|
181
|
+
if (obj.namespace === TOMBSTONE_NAMESPACE) {
|
|
182
|
+
const deleted = obj.deleted;
|
|
183
|
+
// A marker line whose payload is unusable is malformed, NOT a silent
|
|
184
|
+
// no-op: swallowing it would drop a deletion with no diagnostic.
|
|
185
|
+
if (!deleted ||
|
|
186
|
+
typeof deleted.namespace !== 'string' ||
|
|
187
|
+
typeof deleted.key !== 'string' ||
|
|
188
|
+
typeof deleted.at !== 'number') {
|
|
189
|
+
return null;
|
|
190
|
+
}
|
|
191
|
+
return obj;
|
|
192
|
+
}
|
|
193
|
+
return obj;
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* Read + parse an existing artifact, keyed by the (namespace, key) each line is
|
|
197
|
+
* *about* — so a tombstone and the entry it retires occupy one slot and cannot
|
|
198
|
+
* both survive. Missing file → empty.
|
|
199
|
+
*
|
|
200
|
+
* Duplicates within one file are settled by timestamp, newest wins, ties keep
|
|
201
|
+
* the first. That is not hypothetical: an old client re-appends a live line for
|
|
202
|
+
* a key we tombstoned (it keys tombstones under the marker namespace, so the
|
|
203
|
+
* live key looks absent to it). Those re-appended lines carry no `updated_at`,
|
|
204
|
+
* so they stamp 0 and the tombstone stands — while a genuine re-creation from
|
|
205
|
+
* an upgraded client carries a real timestamp and wins.
|
|
206
|
+
*/
|
|
110
207
|
function readArtifact(artifactPath) {
|
|
111
|
-
const
|
|
208
|
+
const lines = new Map();
|
|
209
|
+
const records = new Map();
|
|
112
210
|
let malformed = 0;
|
|
113
211
|
if (!fs.existsSync(artifactPath))
|
|
114
|
-
return {
|
|
212
|
+
return { lines, records, malformed, existed: false };
|
|
115
213
|
const raw = fs.readFileSync(artifactPath, 'utf-8');
|
|
116
214
|
for (const line of raw.split(/\r?\n/)) {
|
|
117
215
|
const trimmed = line.trim();
|
|
118
216
|
if (!trimmed)
|
|
119
217
|
continue;
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
obj = JSON.parse(trimmed);
|
|
123
|
-
}
|
|
124
|
-
catch {
|
|
218
|
+
const parsed = parseLine(trimmed);
|
|
219
|
+
if (!parsed) {
|
|
125
220
|
malformed++;
|
|
126
221
|
continue;
|
|
127
222
|
}
|
|
128
|
-
|
|
129
|
-
|
|
223
|
+
const id = lineId(parsed);
|
|
224
|
+
const record = lineToRecord(parsed);
|
|
225
|
+
const existing = records.get(id);
|
|
226
|
+
// Newest wins, ties keep the first. Built on the merge rule's own
|
|
227
|
+
// comparison basis so the two cannot disagree about which of a live line
|
|
228
|
+
// and a tombstone is later.
|
|
229
|
+
if (existing && recordStamp(existing) >= recordStamp(record))
|
|
130
230
|
continue;
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
// First line for a key wins (first-write-wins inside the file).
|
|
134
|
-
if (!entries.has(mapKey))
|
|
135
|
-
entries.set(mapKey, obj);
|
|
231
|
+
lines.set(id, parsed);
|
|
232
|
+
records.set(id, record);
|
|
136
233
|
}
|
|
137
|
-
return {
|
|
234
|
+
return { lines, records, malformed, existed: true };
|
|
138
235
|
}
|
|
139
|
-
/**
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
return [];
|
|
154
|
-
}
|
|
155
|
-
try {
|
|
156
|
-
if (!hasMemoryEntriesTable(db))
|
|
157
|
-
return [];
|
|
158
|
-
const placeholders = DURABLE_NAMESPACES.map(() => '?').join(',');
|
|
159
|
-
const stmt = db.prepare(`SELECT namespace, key, content, type, tags, created_at FROM memory_entries ` +
|
|
160
|
-
`WHERE namespace IN (${placeholders})`);
|
|
161
|
-
stmt.bind(DURABLE_NAMESPACES.slice());
|
|
162
|
-
const rows = [];
|
|
163
|
-
while (stmt.step()) {
|
|
164
|
-
const r = stmt.getAsObject();
|
|
165
|
-
rows.push({
|
|
166
|
-
namespace: String(r.namespace),
|
|
167
|
-
key: String(r.key),
|
|
168
|
-
content: r.content == null ? '' : String(r.content),
|
|
169
|
-
type: r.type == null ? 'semantic' : String(r.type),
|
|
170
|
-
tags: parseTags(r.tags),
|
|
171
|
-
created_at: r.created_at == null ? undefined : Number(r.created_at),
|
|
172
|
-
// Provenance is filled by the caller (export) — placeholder here.
|
|
173
|
-
provenance: { author: '', source: '', sharedAt: '' },
|
|
174
|
-
});
|
|
175
|
-
}
|
|
176
|
-
stmt.free();
|
|
177
|
-
return rows;
|
|
178
|
-
}
|
|
179
|
-
finally {
|
|
180
|
-
db.close();
|
|
181
|
-
}
|
|
236
|
+
/**
|
|
237
|
+
* Serialise lines to JSONL, sorted by the (namespace, key) each line is about —
|
|
238
|
+
* so a tombstone sorts exactly where the entry it retires used to sit, keeping
|
|
239
|
+
* the git diff line-local and reviewable.
|
|
240
|
+
*/
|
|
241
|
+
function serializeArtifact(lines) {
|
|
242
|
+
const sorted = [...lines].sort((a, b) => {
|
|
243
|
+
const left = splitReconcileId(a[0]);
|
|
244
|
+
const right = splitReconcileId(b[0]);
|
|
245
|
+
return left.namespace === right.namespace
|
|
246
|
+
? left.key.localeCompare(right.key)
|
|
247
|
+
: left.namespace.localeCompare(right.namespace);
|
|
248
|
+
});
|
|
249
|
+
return sorted.map(([, line]) => JSON.stringify(line)).join('\n') + (sorted.length ? '\n' : '');
|
|
182
250
|
}
|
|
251
|
+
/** Parse the DB's JSON-encoded tags column back to the artifact's array form. */
|
|
183
252
|
function parseTags(raw) {
|
|
184
253
|
if (raw == null)
|
|
185
254
|
return undefined;
|
|
186
|
-
if (Array.isArray(raw))
|
|
187
|
-
return raw.map(String);
|
|
188
255
|
try {
|
|
189
|
-
const parsed = JSON.parse(
|
|
256
|
+
const parsed = JSON.parse(raw);
|
|
190
257
|
return Array.isArray(parsed) ? parsed.map(String) : undefined;
|
|
191
258
|
}
|
|
192
259
|
catch {
|
|
193
260
|
return undefined;
|
|
194
261
|
}
|
|
195
262
|
}
|
|
263
|
+
/** Build the artifact line for a local row that is winning the merge. */
|
|
264
|
+
function entryFromPayload(payload, provenance) {
|
|
265
|
+
return {
|
|
266
|
+
namespace: payload.namespace,
|
|
267
|
+
key: payload.key,
|
|
268
|
+
content: payload.content,
|
|
269
|
+
type: payload.type,
|
|
270
|
+
tags: parseTags(payload.tags),
|
|
271
|
+
created_at: payload.createdAt,
|
|
272
|
+
updated_at: payload.updatedAt,
|
|
273
|
+
provenance,
|
|
274
|
+
};
|
|
275
|
+
}
|
|
276
|
+
/** Build the tombstone line for a locally archived row. */
|
|
277
|
+
function tombstoneFromRecord(record, provenance) {
|
|
278
|
+
return {
|
|
279
|
+
namespace: TOMBSTONE_NAMESPACE,
|
|
280
|
+
// Informational for old clients, which key this line under the marker
|
|
281
|
+
// namespace and skip it; the authoritative target is `deleted`.
|
|
282
|
+
key: record.key,
|
|
283
|
+
deleted: { namespace: record.namespace, key: record.key, at: record.deletedAt },
|
|
284
|
+
provenance,
|
|
285
|
+
};
|
|
286
|
+
}
|
|
287
|
+
/** Build the DB row an artifact line becomes. Embeddings are never carried. */
|
|
288
|
+
function payloadFromLine(id, line) {
|
|
289
|
+
return {
|
|
290
|
+
id: createHash('sha1').update(id).digest('hex'),
|
|
291
|
+
namespace: line.namespace,
|
|
292
|
+
key: line.key,
|
|
293
|
+
content: line.content ?? '',
|
|
294
|
+
// An out-of-CHECK-set type would be silently dropped by INSERT OR IGNORE.
|
|
295
|
+
type: coerceType(line.type),
|
|
296
|
+
tags: line.tags ? JSON.stringify(line.tags) : null,
|
|
297
|
+
metadata: JSON.stringify({ provenance: line.provenance, sharedFrom: 'team-artifact' }),
|
|
298
|
+
ownerId: line.provenance?.author || null,
|
|
299
|
+
// created_at/updated_at are INTEGER NOT NULL — never bind null.
|
|
300
|
+
createdAt: typeof line.created_at === 'number' ? line.created_at : Date.now(),
|
|
301
|
+
// The row gets the best timestamp available even when the RECORD compared
|
|
302
|
+
// as 0 (a pre-#1463 line): 0 governs only who wins the merge, while the row
|
|
303
|
+
// itself should carry the most accurate time we have.
|
|
304
|
+
updatedAt: typeof line.updated_at === 'number'
|
|
305
|
+
? line.updated_at
|
|
306
|
+
: typeof line.created_at === 'number'
|
|
307
|
+
? line.created_at
|
|
308
|
+
: Date.now(),
|
|
309
|
+
embedding: null,
|
|
310
|
+
embeddingModel: null,
|
|
311
|
+
embeddingDimensions: null,
|
|
312
|
+
};
|
|
313
|
+
}
|
|
196
314
|
/**
|
|
197
|
-
* Export the local durable slice into the team artifact
|
|
198
|
-
*
|
|
199
|
-
*
|
|
200
|
-
*
|
|
315
|
+
* Export the local durable slice into the team artifact by **reconciling**, not
|
|
316
|
+
* appending: corrections overwrite the artifact line, local archives become
|
|
317
|
+
* tombstones, and entries the artifact has but this machine does not are left
|
|
318
|
+
* strictly alone (they may be a teammate's, or ours-not-yet-imported).
|
|
319
|
+
*
|
|
320
|
+
* The file is rewritten sorted + atomically, and only when something changed —
|
|
321
|
+
* a git-tracked file should not churn its mtime for a no-op run.
|
|
201
322
|
*/
|
|
202
323
|
export function exportTeamArtifact(opts) {
|
|
203
324
|
const projectRoot = opts.projectRoot ?? findProjectRoot();
|
|
204
325
|
const localDbPath = memoryDbPath(projectRoot);
|
|
205
|
-
const
|
|
206
|
-
const
|
|
207
|
-
const
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
326
|
+
const parsedSharedAt = Date.parse(opts.sharedAt);
|
|
327
|
+
const now = opts.now ?? (Number.isNaN(parsedSharedAt) ? Date.now() : parsedSharedAt);
|
|
328
|
+
const ttlMs = opts.tombstoneTtlMs ?? TOMBSTONE_TTL_MS;
|
|
329
|
+
const { lines, records: target, malformed, existed } = readArtifact(opts.artifactPath);
|
|
330
|
+
const provenance = {
|
|
331
|
+
author: resolveAuthor(projectRoot),
|
|
332
|
+
source: resolveSource(),
|
|
333
|
+
sharedAt: opts.sharedAt,
|
|
334
|
+
};
|
|
335
|
+
// Local DB is the source, the artifact the target.
|
|
336
|
+
let records = new Map();
|
|
337
|
+
let payloads = new Map();
|
|
338
|
+
if (fs.existsSync(localDbPath)) {
|
|
339
|
+
let db;
|
|
340
|
+
try {
|
|
341
|
+
db = openDaemonDatabase(localDbPath);
|
|
342
|
+
}
|
|
343
|
+
catch {
|
|
344
|
+
db = null;
|
|
345
|
+
}
|
|
346
|
+
if (db) {
|
|
347
|
+
try {
|
|
348
|
+
// The local DB is the SOURCE here, so it is the side that needs full
|
|
349
|
+
// payloads. Archive retention is applied by the session-start sync, not
|
|
350
|
+
// here: pruning before the artifact write could destroy the evidence
|
|
351
|
+
// for a deletion the write then failed to publish.
|
|
352
|
+
({ records, payloads } = readDurableSnapshot(db, undefined, { withPayloads: true }));
|
|
353
|
+
}
|
|
354
|
+
finally {
|
|
355
|
+
db.close();
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
// #1467 — a local row whose value carries captured tool-call markup is not
|
|
360
|
+
// shared. Dropping it from the SOURCE (rather than the artifact) means it is
|
|
361
|
+
// simply not propagated: `planReconcile` derives deletions from a local
|
|
362
|
+
// tombstone, never from an absent key, so this can't publish a retraction of
|
|
363
|
+
// someone else's line.
|
|
364
|
+
//
|
|
365
|
+
// Gated on `record.content`, which a tombstone does not have, NOT on the
|
|
366
|
+
// payload: archiving leaves `content` intact, so a payload lookup would still
|
|
367
|
+
// see the corrupt body of a row the user already deleted and would suppress
|
|
368
|
+
// its tombstone — making a corrupt line already in the artifact permanently
|
|
369
|
+
// unretractable. Deleting the current entry mid-iteration is well-defined for
|
|
370
|
+
// a Map, so no copy is needed.
|
|
371
|
+
let skippedCorrupt = 0;
|
|
372
|
+
for (const [id, record] of records) {
|
|
373
|
+
if (typeof record.content !== 'string' || !detectToolCallMarkup(record.content))
|
|
374
|
+
continue;
|
|
375
|
+
records.delete(id);
|
|
376
|
+
payloads.delete(id);
|
|
377
|
+
skippedCorrupt++;
|
|
378
|
+
}
|
|
379
|
+
const { actions, summary } = planReconcile(records, target);
|
|
380
|
+
for (const action of actions) {
|
|
381
|
+
if (action.op === 'delete') {
|
|
382
|
+
lines.set(action.id, tombstoneFromRecord(action.record, provenance));
|
|
383
|
+
continue;
|
|
384
|
+
}
|
|
385
|
+
const payload = payloads.get(action.id);
|
|
386
|
+
if (payload)
|
|
387
|
+
lines.set(action.id, entryFromPayload(payload, provenance));
|
|
388
|
+
}
|
|
389
|
+
// Backfill the timestamp on lines this machine holds but that predate #1463.
|
|
390
|
+
// Content-equal lines are never rewritten, so without this they would keep a
|
|
391
|
+
// stamp of 0 forever and lose every future comparison to any real timestamp.
|
|
392
|
+
// Only the timestamp moves — provenance stays with whoever authored the line.
|
|
393
|
+
let backfilled = 0;
|
|
394
|
+
for (const [id, line] of lines) {
|
|
395
|
+
if (isTombstoneLine(line) || typeof line.updated_at === 'number')
|
|
396
|
+
continue;
|
|
397
|
+
const payload = payloads.get(id);
|
|
398
|
+
if (!payload)
|
|
399
|
+
continue;
|
|
400
|
+
lines.set(id, { ...line, updated_at: payload.updatedAt });
|
|
401
|
+
backfilled++;
|
|
402
|
+
}
|
|
403
|
+
const expired = [];
|
|
404
|
+
for (const [id, line] of lines) {
|
|
405
|
+
if (isTombstoneLine(line) && isPrunableTombstone(lineToRecord(line), now, ttlMs))
|
|
406
|
+
expired.push(id);
|
|
215
407
|
}
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
408
|
+
for (const id of expired)
|
|
409
|
+
lines.delete(id);
|
|
410
|
+
const prunedTombstones = expired.length;
|
|
411
|
+
let live = 0;
|
|
412
|
+
let tombstones = 0;
|
|
413
|
+
for (const line of lines.values()) {
|
|
414
|
+
if (isTombstoneLine(line))
|
|
415
|
+
tombstones++;
|
|
416
|
+
else
|
|
417
|
+
live++;
|
|
418
|
+
}
|
|
419
|
+
// Skip the write when the merge changed nothing AND the file already exists:
|
|
420
|
+
// a git-tracked artifact should not show up as modified after a no-op run.
|
|
421
|
+
const changed = actions.length > 0 || prunedTombstones > 0 || backfilled > 0;
|
|
422
|
+
const wrote = changed || !existed;
|
|
423
|
+
if (wrote) {
|
|
424
|
+
fs.mkdirSync(path.dirname(opts.artifactPath), { recursive: true });
|
|
425
|
+
atomicWriteFileSync(opts.artifactPath, serializeArtifact(lines));
|
|
426
|
+
}
|
|
427
|
+
return {
|
|
428
|
+
artifactPath: opts.artifactPath,
|
|
429
|
+
added: summary.inserted,
|
|
430
|
+
updated: summary.updated,
|
|
431
|
+
deleted: summary.deleted,
|
|
432
|
+
resurrected: summary.resurrected,
|
|
433
|
+
unchanged: summary.unchanged,
|
|
434
|
+
keptRemote: summary.keptTargetNewer,
|
|
435
|
+
prunedTombstones,
|
|
436
|
+
backfilled,
|
|
437
|
+
skippedMalformed: malformed,
|
|
438
|
+
skippedCorrupt,
|
|
439
|
+
total: live,
|
|
440
|
+
tombstones,
|
|
441
|
+
wrote,
|
|
442
|
+
};
|
|
219
443
|
}
|
|
220
444
|
/**
|
|
221
|
-
* Import-merge the team artifact into the local durable namespaces
|
|
222
|
-
*
|
|
223
|
-
*
|
|
224
|
-
*
|
|
225
|
-
*
|
|
445
|
+
* Import-merge the team artifact into the local durable namespaces, applying
|
|
446
|
+
* the same reconciliation rule with the artifact as source. Corrections update
|
|
447
|
+
* the local row, tombstones archive it, and local-only rows are never touched.
|
|
448
|
+
*
|
|
449
|
+
* Embeddings are left null on insert and cleared on update — the artifact
|
|
450
|
+
* carries no vectors, so keeping the old one would leave a corrected row
|
|
451
|
+
* findable under its previous meaning. The daemon's index pass refills them.
|
|
452
|
+
*
|
|
453
|
+
* No-op (zero rows) when the artifact is absent. Never throws on a malformed
|
|
454
|
+
* line; those are counted and skipped.
|
|
226
455
|
*/
|
|
227
456
|
export function importTeamArtifact(opts) {
|
|
228
457
|
const projectRoot = opts.projectRoot ?? findProjectRoot();
|
|
@@ -230,86 +459,99 @@ export function importTeamArtifact(opts) {
|
|
|
230
459
|
const report = {
|
|
231
460
|
artifactPath: opts.artifactPath,
|
|
232
461
|
imported: 0,
|
|
462
|
+
updated: 0,
|
|
463
|
+
deleted: 0,
|
|
464
|
+
resurrected: 0,
|
|
465
|
+
keptLocal: 0,
|
|
233
466
|
considered: 0,
|
|
234
467
|
skippedMalformed: 0,
|
|
235
468
|
skippedNonDurable: 0,
|
|
469
|
+
skippedCorrupt: 0,
|
|
236
470
|
};
|
|
237
|
-
const {
|
|
471
|
+
const { lines, records: parsed, malformed } = readArtifact(opts.artifactPath);
|
|
238
472
|
report.skippedMalformed = malformed;
|
|
239
|
-
if (
|
|
473
|
+
if (lines.size === 0)
|
|
474
|
+
return report;
|
|
475
|
+
const source = new Map();
|
|
476
|
+
for (const [id, line] of lines) {
|
|
477
|
+
const record = parsed.get(id);
|
|
478
|
+
// The artifact is hand-editable by design — only durable namespaces belong
|
|
479
|
+
// in the local durable slice. A tombstone resolves to its inner namespace,
|
|
480
|
+
// so it passes this check while its marker namespace keeps old clients from
|
|
481
|
+
// acting on it.
|
|
482
|
+
if (!isDurableNamespace(record.namespace)) {
|
|
483
|
+
report.skippedNonDurable++;
|
|
484
|
+
continue;
|
|
485
|
+
}
|
|
486
|
+
// #1467 — never write captured tool-call markup into the local store, even
|
|
487
|
+
// when a teammate on an older moflo already shared it. Tombstones carry no
|
|
488
|
+
// content and are always applied: a deletion must still propagate.
|
|
489
|
+
if (!isTombstoneLine(line) && detectToolCallMarkup(line.content)) {
|
|
490
|
+
report.skippedCorrupt++;
|
|
491
|
+
continue;
|
|
492
|
+
}
|
|
493
|
+
report.considered++;
|
|
494
|
+
source.set(id, record);
|
|
495
|
+
}
|
|
496
|
+
if (source.size === 0)
|
|
240
497
|
return report;
|
|
241
498
|
fs.mkdirSync(path.dirname(localDbPath), { recursive: true });
|
|
242
499
|
const db = openDaemonDatabase(localDbPath);
|
|
243
|
-
let insertStmt = null;
|
|
244
500
|
try {
|
|
245
|
-
db
|
|
246
|
-
|
|
247
|
-
//
|
|
248
|
-
|
|
249
|
-
//
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
report.skippedNonDurable++;
|
|
259
|
-
continue;
|
|
260
|
-
}
|
|
261
|
-
report.considered++;
|
|
262
|
-
const id = createHash('sha1').update(entryMapKey(entry.namespace, entry.key)).digest('hex');
|
|
263
|
-
const metadata = JSON.stringify({ provenance: entry.provenance, sharedFrom: 'team-artifact' });
|
|
264
|
-
// created_at/updated_at are INTEGER NOT NULL — never bind null (INSERT OR
|
|
265
|
-
// IGNORE would silently drop the row on the constraint violation).
|
|
266
|
-
const ts = typeof entry.created_at === 'number' ? entry.created_at : Date.now();
|
|
267
|
-
insertStmt.bind([
|
|
268
|
-
id,
|
|
269
|
-
entry.key,
|
|
270
|
-
entry.namespace,
|
|
271
|
-
entry.content ?? '',
|
|
272
|
-
coerceType(entry.type), // out-of-CHECK-set type would be silently dropped otherwise
|
|
273
|
-
null, // embedding — regenerated by the daemon's index pass
|
|
274
|
-
null, // embedding_model
|
|
275
|
-
null, // embedding_dimensions
|
|
276
|
-
entry.tags ? JSON.stringify(entry.tags) : null,
|
|
277
|
-
metadata,
|
|
278
|
-
entry.provenance?.author || null,
|
|
279
|
-
ts,
|
|
280
|
-
ts,
|
|
281
|
-
'active',
|
|
282
|
-
]);
|
|
283
|
-
insertStmt.step();
|
|
284
|
-
if (db.getRowsModified() > 0)
|
|
285
|
-
report.imported++;
|
|
286
|
-
insertStmt.reset();
|
|
287
|
-
}
|
|
288
|
-
db.run('COMMIT');
|
|
289
|
-
}
|
|
290
|
-
catch (e) {
|
|
291
|
-
try {
|
|
292
|
-
db.run('ROLLBACK');
|
|
293
|
-
}
|
|
294
|
-
catch {
|
|
295
|
-
/* best-effort — the close() below also discards an open transaction */
|
|
296
|
-
}
|
|
297
|
-
throw e;
|
|
501
|
+
const { records: target } = readDurableSnapshot(db);
|
|
502
|
+
const { actions, summary } = planReconcile(source, target);
|
|
503
|
+
// Payloads are built only for the actions that need one. In the steady
|
|
504
|
+
// state the plan is empty, and hashing + re-encoding every line of a
|
|
505
|
+
// 1,300-entry artifact on every session start would be pure waste.
|
|
506
|
+
const payloads = new Map();
|
|
507
|
+
for (const action of actions) {
|
|
508
|
+
if (action.op === 'delete')
|
|
509
|
+
continue;
|
|
510
|
+
const line = lines.get(action.id);
|
|
511
|
+
if (!line || isTombstoneLine(line))
|
|
512
|
+
continue;
|
|
513
|
+
payloads.set(action.id, payloadFromLine(action.id, line));
|
|
298
514
|
}
|
|
515
|
+
const applied = applyDurableActions(db, actions, payloads);
|
|
516
|
+
report.imported = applied.inserted;
|
|
517
|
+
report.updated = applied.updated;
|
|
518
|
+
report.deleted = applied.archived;
|
|
519
|
+
report.resurrected = applied.resurrected;
|
|
520
|
+
report.keptLocal = summary.keptTargetNewer;
|
|
299
521
|
}
|
|
300
522
|
finally {
|
|
301
|
-
if (insertStmt) {
|
|
302
|
-
try {
|
|
303
|
-
insertStmt.free();
|
|
304
|
-
}
|
|
305
|
-
catch {
|
|
306
|
-
/* best-effort cleanup */
|
|
307
|
-
}
|
|
308
|
-
}
|
|
309
523
|
db.close();
|
|
310
524
|
}
|
|
311
525
|
return report;
|
|
312
526
|
}
|
|
527
|
+
/**
|
|
528
|
+
* The dominant line ending already in a file. Both writers below rewrite whole
|
|
529
|
+
* files they did not author, so they must hand back the style they were given —
|
|
530
|
+
* splitting on `/\r?\n/` and re-joining with `\n` silently converts every
|
|
531
|
+
* unrelated line in a CRLF checkout, which is the opposite of the narrow,
|
|
532
|
+
* one-line edit these functions promise (Rule #1).
|
|
533
|
+
*/
|
|
534
|
+
function detectEol(raw) {
|
|
535
|
+
const newlines = (raw.match(/\n/g) ?? []).length;
|
|
536
|
+
const crlf = (raw.match(/\r\n/g) ?? []).length;
|
|
537
|
+
return crlf > newlines - crlf ? '\r\n' : '\n';
|
|
538
|
+
}
|
|
539
|
+
/**
|
|
540
|
+
* Render one path as a literal gitattributes pattern.
|
|
541
|
+
*
|
|
542
|
+
* The artifact path is caller-supplied (`flo memory team-export --to ...`), so
|
|
543
|
+
* it can carry whitespace or glob metacharacters. Unescaped, `team notes.jsonl`
|
|
544
|
+
* parses as the pattern `team` plus garbage, and a `*` in the name would match
|
|
545
|
+
* files the user never named. Glob metacharacters are escaped first; the result
|
|
546
|
+
* is then C-quoted if it contains whitespace, which doubles the backslashes the
|
|
547
|
+
* escape pass added — that is correct, since git un-quotes before globbing.
|
|
548
|
+
*/
|
|
549
|
+
function gitattributesPattern(rel) {
|
|
550
|
+
const pattern = `/${rel.replace(/([\\*?[\]])/g, '\\$1')}`;
|
|
551
|
+
if (!/[\s"]/.test(pattern))
|
|
552
|
+
return pattern;
|
|
553
|
+
return `"${pattern.replace(/(["\\])/g, '\\$1')}"`;
|
|
554
|
+
}
|
|
313
555
|
/**
|
|
314
556
|
* Ensure a git-tracked shared artifact is actually trackable. Once `.moflo/` is
|
|
315
557
|
* gitignored, git won't descend into it to re-include a child — the canonical
|
|
@@ -333,7 +575,9 @@ export function ensureSharedArtifactTracked(projectRoot, artifactAbsPath) {
|
|
|
333
575
|
const relDir = rel.slice(0, rel.lastIndexOf('/') + 1);
|
|
334
576
|
const negation = `!/${relDir}`;
|
|
335
577
|
const existed = fs.existsSync(gitignorePath);
|
|
336
|
-
const
|
|
578
|
+
const raw = existed ? fs.readFileSync(gitignorePath, 'utf-8') : '';
|
|
579
|
+
const eol = existed ? detectEol(raw) : '\n';
|
|
580
|
+
const lines = existed ? raw.split(/\r?\n/) : [];
|
|
337
581
|
const isBareMoflo = (t) => t === '.moflo/' || t === '/.moflo/' || t === '.moflo' || t === '/.moflo';
|
|
338
582
|
const isContentsRule = (t) => t === '.moflo/*' || t === '/.moflo/*';
|
|
339
583
|
let changed = false;
|
|
@@ -373,8 +617,63 @@ export function ensureSharedArtifactTracked(projectRoot, artifactAbsPath) {
|
|
|
373
617
|
}
|
|
374
618
|
if (!changed)
|
|
375
619
|
return 'unchanged';
|
|
376
|
-
const content = next.join(
|
|
620
|
+
const content = next.join(eol).replace(/(\r?\n)+$/, '') + eol;
|
|
377
621
|
atomicWriteFileSync(gitignorePath, content);
|
|
378
622
|
return existed ? 'updated' : 'created';
|
|
379
623
|
}
|
|
624
|
+
/**
|
|
625
|
+
* Pin the shared artifact to LF in the consumer's `.gitattributes`.
|
|
626
|
+
*
|
|
627
|
+
* The artifact is a git-tracked file in someone else's repo, and moflo always
|
|
628
|
+
* writes it with `\n`. On a Windows checkout with `core.autocrlf=true` git
|
|
629
|
+
* hands back CRLF, so every export rewrites the whole file and — far worse — a
|
|
630
|
+
* git merge of two divergent artifacts conflicts on EVERY line. That destroys
|
|
631
|
+
* the merge-friendliness that made JSONL the format in the first place, and it
|
|
632
|
+
* only bites the platform least likely to be running CI.
|
|
633
|
+
*
|
|
634
|
+
* One narrowly-scoped line for one file; never touches unrelated patterns.
|
|
635
|
+
* Idempotent — an existing rule for this path is left exactly as written, so a
|
|
636
|
+
* consumer who tuned it keeps their version.
|
|
637
|
+
*
|
|
638
|
+
* Gitattributes patterns always use `/` regardless of OS (Rule #1), so the
|
|
639
|
+
* path is POSIX-normalised here.
|
|
640
|
+
*/
|
|
641
|
+
export function ensureSharedArtifactEol(projectRoot, artifactAbsPath) {
|
|
642
|
+
const rel = path.relative(projectRoot, artifactAbsPath).split(path.sep).join('/');
|
|
643
|
+
// Outside the project entirely (an absolute artifact on a shared drive) —
|
|
644
|
+
// there is no repo of ours to annotate.
|
|
645
|
+
if (rel.startsWith('..') || path.isAbsolute(rel))
|
|
646
|
+
return 'unchanged';
|
|
647
|
+
const attributesPath = path.join(projectRoot, '.gitattributes');
|
|
648
|
+
const pattern = gitattributesPattern(rel);
|
|
649
|
+
const existed = fs.existsSync(attributesPath);
|
|
650
|
+
const raw = existed ? fs.readFileSync(attributesPath, 'utf-8') : '';
|
|
651
|
+
const eol = existed ? detectEol(raw) : '\n';
|
|
652
|
+
const lines = existed ? raw.split(/\r?\n/) : [];
|
|
653
|
+
// macOS APFS and Windows NTFS are case-insensitive by default, so a rule
|
|
654
|
+
// differing only in case there names the SAME file and must count as already
|
|
655
|
+
// ruled. On a case-sensitive filesystem it names a different file, and
|
|
656
|
+
// skipping ours would leave the artifact unpinned — so the fold is applied
|
|
657
|
+
// only where the filesystem actually folds (Rule #1).
|
|
658
|
+
const caseInsensitive = process.platform === 'win32' || process.platform === 'darwin';
|
|
659
|
+
const fold = (v) => (caseInsensitive ? v.toLowerCase() : v);
|
|
660
|
+
const candidates = [fold(pattern), fold(`/${rel}`), fold(rel)];
|
|
661
|
+
// Any existing rule naming this path wins, whatever it says.
|
|
662
|
+
const alreadyRuled = lines.some((line) => {
|
|
663
|
+
const trimmed = fold(line.trim());
|
|
664
|
+
if (!trimmed || trimmed.startsWith('#'))
|
|
665
|
+
return false;
|
|
666
|
+
return candidates.some((c) => trimmed === c || trimmed.startsWith(`${c} `) || trimmed.startsWith(`${c}\t`));
|
|
667
|
+
});
|
|
668
|
+
if (alreadyRuled)
|
|
669
|
+
return 'unchanged';
|
|
670
|
+
const next = [...lines];
|
|
671
|
+
if (next.length && next[next.length - 1].trim() !== '')
|
|
672
|
+
next.push('');
|
|
673
|
+
next.push('# moflo team-shared learnings: LF so the artifact stays diffable and merge-friendly');
|
|
674
|
+
next.push(`${pattern} text eol=lf`);
|
|
675
|
+
const content = next.join(eol).replace(/(\r?\n)+$/, '') + eol;
|
|
676
|
+
atomicWriteFileSync(attributesPath, content);
|
|
677
|
+
return existed ? 'updated' : 'created';
|
|
678
|
+
}
|
|
380
679
|
//# sourceMappingURL=team-artifact-sync.js.map
|