@awebai/oats 0.23.0 → 0.23.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/README.md +48 -18
  2. package/capabilities/oats-okf/agents/memory-harvest/AGENTS.md +18 -24
  3. package/capabilities/oats-okf/agents/memory-harvest/soul.yaml +2 -2
  4. package/capabilities/oats-okf/bin/oats-okf.mjs +105 -517
  5. package/capabilities/oats-okf/injects/okf.md +32 -67
  6. package/capabilities/oats-okf/lib/config.mjs +112 -0
  7. package/capabilities/oats-okf/lib/inspection.mjs +96 -0
  8. package/capabilities/oats-okf/lib/io.mjs +103 -0
  9. package/capabilities/oats-okf/lib/migration.mjs +116 -0
  10. package/capabilities/oats-okf/lib/sources.mjs +238 -0
  11. package/capabilities/oats-okf/lib/stores.mjs +331 -0
  12. package/capabilities/oats-okf/lib/worker.mjs +352 -0
  13. package/capabilities/oats-okf/oats.json +23 -7
  14. package/capabilities/oats-okf/schemas/okf-base.schema.json +46 -0
  15. package/capabilities/oats-okf/schemas/okf-bindings.schema.json +112 -0
  16. package/capabilities/oats-okf/schemas/okf-soul.schema.json +37 -0
  17. package/capabilities/oats-okf/skills/memory-harvest/SKILL.md +263 -140
  18. package/capabilities/oats-okf/skills/okf/SKILL.md +13 -4
  19. package/docs/capabilities.md +14 -3
  20. package/docs/configuration.md +11 -1
  21. package/docs/design/okf-mirror-provenance.md +105 -0
  22. package/docs/desktop-cli-api.md +59 -10
  23. package/docs/first-team-demo.md +6 -1
  24. package/docs/first-team.md +151 -115
  25. package/docs/integrations.md +42 -42
  26. package/docs/knowledge-capability-authoring.md +10 -7
  27. package/docs/knowledge-migration.md +138 -0
  28. package/docs/knowledge.md +316 -129
  29. package/docs/layers.md +57 -62
  30. package/docs/migration-from-oas.md +7 -1
  31. package/docs/packages.md +26 -2
  32. package/docs/release-notes/v0.23.1.md +97 -0
  33. package/docs/schedules.md +42 -3
  34. package/docs/souls-and-instances.md +55 -48
  35. package/package-catalog.json +6 -1
  36. package/package.json +1 -1
  37. package/capabilities/oats-okf/lib/harvest-branch.mjs +0 -43
@@ -0,0 +1,238 @@
1
+ import { randomUUID } from 'node:crypto';
2
+ import { fs, join, dirname, resolve, safePath, readJSON, save, atomic, materialize, hash, withLock, oats, fail, tree, overlaps, syncDir } from './io.mjs';
3
+ import { loadBindings, declaration, metadata, resolveNodes, bindingFingerprint, settings, validateBindings } from './config.mjs';
4
+ import { stageBase } from './stores.mjs';
5
+ export const markerPath = home => join(home,'.okf-source.json');
6
+ export const statusPath = source => join(dirname(source.file),'status.json');
7
+ export const saveStatus = (source,status) => save(statusPath(source),status);
8
+ export function updateStatus(source, mutate) {
9
+ return withLock(join(dirname(source.file),'status.lock'),()=>{
10
+ const status=loadStatus(source);mutate(status);saveStatus(source,status);return status;
11
+ });
12
+ }
13
+ function saveCapture(source, status) {
14
+ updateStatus(source,current=>{
15
+ current.captured=status.captured;
16
+ if(status.launchObserved!==undefined) current.launchObserved=status.launchObserved;
17
+ if(status.lastCapture) current.lastCapture=status.lastCapture;
18
+ if(status.retired) {current.retired=true;current.retiredAt=status.retiredAt;current.auto=current.auto && status.auto;}
19
+ });
20
+ }
21
+ export const loadStatus = source => readJSON(statusPath(source));
22
+ export function loadSource(file) {
23
+ safePath(file); const s=readJSON(file);
24
+ if(s.version!==1 || !/^[0-9a-f-]{36}$/.test(s.id) || resolve(file)!==join(s.bindings.stateDir,'sources',s.id,'source.json')) fail('E_SOURCE','invalid source descriptor path/identity');
25
+ const {file:bindingsFile,...bindingsDoc}=s.bindings;
26
+ const checked=validateBindings(bindingsDoc,bindingsFile,{sourceHome:s.home,sourceWork:s.work});
27
+ if(bindingFingerprint(checked)!==s.bindingFingerprint) fail('E_SOURCE','frozen bindings fingerprint mismatch');
28
+ return {...s,file};
29
+ }
30
+ export function homeSource(home) { const m=readJSON(markerPath(home)); const s=loadSource(m.source); if(s.id!==m.id || s.home!==home) fail('E_SOURCE','home identity does not match durable source');return s; }
31
+ export function service(home) {
32
+ if(fs.existsSync(join(home,'instance.json'))) return readJSON(join(home,'instance.json')).kind==='capability';
33
+ return process.env.OATS_KIND==='capability';
34
+ }
35
+ // Control files live at the view root; arbitrary legal aliases live ONLY in
36
+ // bases/. Receipts use paths relative to the view so moving a prepared view
37
+ // into its final location cannot invalidate its navigation.
38
+ export function views(bindings, decl, target) {
39
+ target=safePath(target); if(fs.existsSync(target)) fail('E_VIEW','view exists; use a new immutable view destination');
40
+ const all={}; const receipts={};
41
+ fs.mkdirSync(dirname(target),{recursive:true});
42
+ const pending=fs.mkdtempSync(join(dirname(target),'.okf-view-'));
43
+ try {
44
+ for(const [alias,base] of Object.entries(bindings.bases)) {
45
+ const scratch=fs.mkdtempSync(join(bindings.stateDir,'read-'));
46
+ try {
47
+ const staged=stageBase(base,join(scratch,'base'));
48
+ all[alias]=staged.meta;
49
+ const path=`bases/${alias}`;
50
+ materialize(join(pending,path),staged.files);
51
+ receipts[alias]={path,id:base.id,digest:staged.digest,head:staged.head || null,nodes:staged.meta.nodes};
52
+ } finally { fs.rmSync(scratch,{recursive:true,force:true}); }
53
+ }
54
+ resolveNodes(decl,bindings,all);
55
+ save(join(pending,'view.json'),{version:1,at:new Date().toISOString(),bases:receipts,owns:decl.owns,reads:decl.reads});
56
+ // Never expose a partial view or remove/replace a caller's existing view.
57
+ if(fs.existsSync(target)) fail('E_VIEW','view exists; use a new immutable view destination');
58
+ fs.renameSync(pending,target);syncDir(dirname(target));
59
+ return receipts;
60
+ } finally { fs.rmSync(pending,{recursive:true,force:true}); }
61
+ }
62
+ const registrationView = source => join(source.home,`.okf-view-${source.id}`);
63
+ function finishRegistration(source) {
64
+ const pending=safePath(registrationView(source)),target=safePath(join(source.home,'knowledge'));
65
+ if(fs.existsSync(pending)) {
66
+ if(fs.existsSync(target)) fail('E_VIEW','knowledge already exists; refusing to replace it with the registered view');
67
+ fs.renameSync(pending,target);syncDir(source.home);
68
+ }
69
+ // A durable home pointer precedes publication. Failures after it was saved
70
+ // resume this same source/view; they never reset captured evidence or IDs.
71
+ const receipt=readJSON(join(target,'view.json'));
72
+ if(source.acceptedView && JSON.stringify(receipt.bases)!==JSON.stringify(source.acceptedView)) fail('E_VIEW','registered accepted view receipt differs; preserve it and inspect');
73
+ for(const [p,text] of [['STATE.md','# Working state\n\n# Task\n\n# Next\n'],['log.md','# Instance log\n']]) if(!fs.existsSync(join(source.home,p))) atomic(join(source.home,p),text);
74
+ fs.mkdirSync(join(source.home,'notes'),{recursive:true});
75
+ scheduleSource(source);return source;
76
+ }
77
+ export function register(home) {
78
+ home=safePath(home);
79
+ if(service(home)) return {skipped:'service'};
80
+ if(fs.existsSync(markerPath(home))) return finishRegistration(homeSource(home));
81
+ safePath(join(home,'knowledge'));
82
+ if(fs.existsSync(join(home,'knowledge'))) fail('E_VIEW','unregistered knowledge view exists; preserve it and inspect before registering');
83
+ if(['.okf-harvest-record.json','.okf-harvest-record.next.json'].some(p=>fs.existsSync(join(home,p))) && !fs.existsSync(join(home,'.okf-v1-migration.json'))) fail('E_MIGRATION','legacy source watermarks require explicit oats okf migrate --source-home PATH before v2 registration; no cursor is silently trusted');
84
+ const meta=fs.existsSync(join(home,'instance.json'))?readJSON(join(home,'instance.json')):{};
85
+ const soul=fs.realpathSync(process.env.OATS_SOUL || join(home,'soul'));
86
+ const work=fs.existsSync(join(home,'work'))?fs.realpathSync(join(home,'work')):join(home,'work');
87
+ const decl=declaration(soul);
88
+ const bindings=loadBindings(undefined,{sourceHome:home,sourceWork:work});
89
+ const context=fs.realpathSync(process.env.OATS_CONTEXT || meta.repo || fail('E_CONFIG','source requires durable config context'));
90
+ if(overlaps(home,context) && context.startsWith(home)) fail('E_PATH','config context cannot be in disposable home');
91
+ const agent=process.env.OATS_AGENT || meta.agent;
92
+ const instance=process.env.OATS_INSTANCE || meta.instance;
93
+ if(!agent || !instance) fail('E_SOURCE','source instance/agent required');
94
+ fs.mkdirSync(bindings.stateDir,{recursive:true,mode:0o700});
95
+ const ownersFile=join(bindings.stateDir,'owners.json');
96
+ withLock(join(bindings.stateDir,'owners.lock'),()=>{
97
+ const owners=fs.existsSync(ownersFile)?readJSON(ownersFile):{};
98
+ if(Object.hasOwn(owners,decl.owner) && owners[decl.owner]!==soul) fail('E_OWNER','stable owner ID already identifies a different soul in this state namespace');
99
+ owners[decl.owner]=soul;save(ownersFile,owners);
100
+ });
101
+ const id=randomUUID(); const dir=join(bindings.stateDir,'sources',id);
102
+ // Copy only the role document, never instance.json wholesale, launch recipes,
103
+ // environment, credentials, source worktree, or third-party message stores.
104
+ const roleFile=safePath(join(soul,'AGENTS.md'));
105
+ const role=fs.existsSync(roleFile)?fs.readFileSync(roleFile,'utf8'):'';
106
+ if(Buffer.byteLength(role)>128*1024) fail('E_SOURCE','role document exceeds 128KiB; provide a concise role before registering');
107
+ const source={version:1,id,home,work,context,agent,instance,owner:decl.owner,decl,role,bindings,bindingFingerprint:bindingFingerprint(bindings),execution:{runtime:settings()['harvest-runtime']||'pi',model:settings()['harvest-model']||null},created:new Date().toISOString()};
108
+ const file=join(dir,'source.json');
109
+ const pending=registrationView(source);
110
+ fs.mkdirSync(dir,{recursive:true,mode:0o700});
111
+ try {
112
+ source.acceptedView=views(bindings,decl,pending);
113
+ source.acceptedNodes=Object.fromEntries(Object.entries(source.acceptedView).map(([alias,r])=>[alias,r.nodes]));
114
+ save(file,source);
115
+ save(join(dir,'status.json'),{version:1,captured:{notes:[],threads:{},inputs:[]},processed:[],delivered:{},accepted:{},retired:false,auto:true,activeRun:null});
116
+ save(markerPath(home),{version:1,id,source:file});
117
+ } catch(e) {
118
+ // Until the pointer is durable no capture or schedule can reference these
119
+ // files. Leave an installed pointer's state intact even if fsync failed.
120
+ if(!fs.existsSync(markerPath(home))) {
121
+ fs.rmSync(pending,{recursive:true,force:true});
122
+ fs.rmSync(dir,{recursive:true,force:true});
123
+ }
124
+ throw e;
125
+ }
126
+ return finishRegistration({...source,file});
127
+ }
128
+ function enqueue(source,status,payload) {
129
+ const id=hash(payload);const path=join(dirname(source.file),'inputs',`${id}.json`);
130
+ if(!fs.existsSync(path)) save(path,payload);
131
+ if(!status.captured.inputs.includes(id)) status.captured.inputs.push(id);
132
+ return id;
133
+ }
134
+ export function input(source,id) {
135
+ if(!/^[0-9a-f]{64}$/.test(id)) fail('E_INPUT','bad input identity');
136
+ const value=readJSON(join(dirname(source.file),'inputs',`${id}.json`));
137
+ if(hash(value)!==id) fail('E_INPUT','durable evidence hash mismatch');return value;
138
+ }
139
+ export function capture(source,{final=false,deadlineMs=85000}={}) {
140
+ return withLock(join(dirname(source.file),'capture.lock'),()=>{
141
+ const status=loadStatus(source);
142
+ if(status.retired) return {status:'complete',complete:true,retired:true};
143
+ if(!fs.existsSync(markerPath(source.home))) fail('E_SOURCE','source home gone without final capture; existing evidence is retained');
144
+ if(homeSource(source.home).id!==source.id) fail('E_SOURCE','source name reused');
145
+ const start=Date.now(); const meta=fs.existsSync(join(source.home,'instance.json'))?readJSON(join(source.home,'instance.json')):{};
146
+ const noLaunch=meta.launched!==true;status.launchObserved=!noLaunch;
147
+ // Notes AND record, every pass. Note content versions remain captured even
148
+ // when the live file is rewritten while a worker is judging a prior version.
149
+ const notes=join(source.home,'notes');
150
+ if(fs.existsSync(notes)) for(const [name,b64] of Object.entries(tree(notes))) {
151
+ if(!name.endsWith('.md')) continue;
152
+ const text=Buffer.from(b64,'base64').toString('utf8');const key=hash({name,text});
153
+ if(status.captured.notes.includes(key)) continue;
154
+ enqueue(source,status,{version:1,kind:'note',name,contentHash:hash(text),text});
155
+ status.captured.notes.push(key);saveCapture(source,status);
156
+ }
157
+ let report;
158
+ try {
159
+ report=oats(['capture','--home',source.home,'--quiet'],source.context,{native:true,timeout:Math.min(deadlineMs,60000)});
160
+ if(!Array.isArray(report.sessions)) fail('E_CAPTURE','capture response has no sessions');
161
+ for(const session of report.sessions) {
162
+ if(!session.thread || !session.lastTurnId) continue;
163
+ let after=status.captured.threads[session.thread] || null;
164
+ while(after!==session.lastTurnId) {
165
+ if(Date.now()-start>deadlineMs) fail('E_CAPTURE','capture deadline: backlog preserved; retire must retry');
166
+ const args=['recall','--thread',session.thread,'--until',session.lastTurnId,'--limit','60','--json','--ids-only'];
167
+ if(after) args.push('--after',after);
168
+ const remainingTime=()=>{
169
+ const remaining=deadlineMs-(Date.now()-start);
170
+ if(remaining<=0) fail('E_CAPTURE','capture deadline: backlog preserved; retire must retry');
171
+ return Math.max(1000,remaining);
172
+ };
173
+ let plan;
174
+ try { plan=oats(args,source.context,{native:true,timeout:remainingTime()}); }
175
+ catch(e) { if(after && /--after: no turn/.test(e.message)) {after=null;continue;} throw e; }
176
+ if(!Array.isArray(plan.turns) || !plan.turns.length || plan.turns.length>60) fail('E_CAPTURE','capture/recall boundaries disagree');
177
+ // Plan BEFORE requesting text. Native bytes describe each turn in
178
+ // pretty JSON. Nested response indentation adds bytes: reserve 3x
179
+ // that estimate plus envelope space, always below the 16MiB pipe.
180
+ // The 96k target is soft only for one individually legal turn.
181
+ let offset=0;
182
+ while(offset<plan.turns.length) {
183
+ const window=[];let bytes=0;
184
+ for(const t of plan.turns.slice(offset)) {
185
+ if(typeof t.id!=='string' || !t.id || t.thread!==session.thread || t.kind!=='session' || !Number.isSafeInteger(t.bytes) || t.bytes<=0) fail('E_CAPTURE','invalid ids-only record metadata');
186
+ if(window.length && bytes+t.bytes>96000) break;
187
+ if(3*(bytes+t.bytes)+4096>16*1024*1024) fail('E_CAPTURE','single captured turn exceeds bounded transport; input retained at source, manual intervention required');
188
+ window.push(t);bytes+=t.bytes;
189
+ }
190
+ const until=window.at(-1).id;
191
+ const read=['recall','--thread',session.thread,'--until',until,'--limit',String(window.length),'--json'];
192
+ if(after) read.push('--after',after);
193
+ const full=oats(read,source.context,{native:true,timeout:remainingTime()});
194
+ if(!Array.isArray(full.turns) || full.turns.length!==window.length || full.remaining!==0) fail('E_CAPTURE','planned record window changed; retain source and retry');
195
+ const turns=full.turns.map((t,i)=>{
196
+ const turn={id:t.id,ts:t.ts,thread:t.thread,kind:t.kind,source:t.source,text:t.text};
197
+ if(['id','ts','thread','kind','source'].some(k=>turn[k]!==window[i][k]) || !Array.isArray(turn.text) || turn.text.some(p=>typeof p.role!=='string' || typeof p.text!=='string')) fail('E_CAPTURE','unsupported or changed captured session record');
198
+ if(Buffer.byteLength(JSON.stringify(turn))>1024*1024) fail('E_CAPTURE','single captured turn exceeds 1MiB; input retained at source, manual intervention required');
199
+ return turn;
200
+ });
201
+ // Commit evidence before its cursor; failures retain earlier windows.
202
+ enqueue(source,status,{version:1,kind:'record',thread:session.thread,after,until,turns});
203
+ after=until;status.captured.threads[session.thread]=after;saveCapture(source,status);
204
+ offset+=window.length;
205
+ }
206
+ if(plan.remaining===0 && after!==session.lastTurnId) fail('E_CAPTURE','final record boundary missing');
207
+ }
208
+ }
209
+ status.lastCapture={status:report.status,complete:report.complete===true,ignored:report.ignored||0,at:new Date().toISOString()};
210
+ if(report.complete!==true) fail('E_CAPTURE',`capture ${report.status || 'uncertified'}: retain source and retry`);
211
+ if(final) {status.retired=true;status.auto=status.auto && !noLaunch;status.retiredAt=new Date().toISOString();}
212
+ saveCapture(source,status);return {...status.lastCapture,inputs:status.captured.inputs.length};
213
+ } catch(e) {
214
+ status.lastCapture={status:'incomplete',complete:false,error:e.message,at:new Date().toISOString()}; saveCapture(source,status);throw e;
215
+ }
216
+ });
217
+ }
218
+ export function scheduleSource(source) {
219
+ const spec={id:`okf-${source.id}`,kind:'command',enabled:loadStatus(source).auto,cron:source.bindings.cron,tz:source.bindings.tz,cwd:source.context,argv:['oats','okf','run-source','--source',source.file,'--soul',source.agent,'--json']};
220
+ const file=join(dirname(source.file),'schedule.json');
221
+ try {
222
+ save(file,spec);
223
+ let result;
224
+ try {result=oats(['schedule','add',spec.id,'--file',file,'--dir',source.context,'--json'],source.context);}
225
+ catch(e) {
226
+ // Never overwrite a colliding job or re-enable an operator-disabled job.
227
+ // Retry after an uncertain add must verify the actual definition, not a
228
+ // local receipt. A deleted definition is recreated by the add above.
229
+ if(e.code!=='E_SCHEDULE_EXISTS') throw e;
230
+ result=oats(['schedule','show',spec.id,'--dir',source.context,'--json'],source.context);
231
+ }
232
+ const actual=result?.schedule;
233
+ if(!actual || typeof actual.enabled!=='boolean' || ['id','kind','cron','tz','cwd','argv'].some(k=>JSON.stringify(actual[k])!==JSON.stringify(spec[k]))) fail('E_SCHEDULE','source schedule definition differs; inspect and repair explicitly');
234
+ updateStatus(source,status=>{status.schedule={id:spec.id,status:'ready',result};});return result;
235
+ } catch(e) {
236
+ updateStatus(source,status=>{status.schedule={...(status.schedule || {}),id:spec.id,status:'failed',error:e.message};});throw e;
237
+ }
238
+ }
@@ -0,0 +1,331 @@
1
+ import { createHash } from 'node:crypto';
2
+ import { fileURLToPath } from 'node:url';
3
+ import { fs, join, dirname, safePath, readJSON, save, atomic, tree, materialize, digest, hash, withLock, exec, cleanEnv, fail, relPath, overlaps, resolve } from './io.mjs';
4
+ import { metadata, noGit } from './config.mjs';
5
+ const validator = fileURLToPath(new URL('../skills/okf/scripts/okf-validate.mjs', import.meta.url));
6
+ // Never let local replace refs reinterpret frozen OIDs, including inside Git's
7
+ // transport subprocesses. Override even an explicitly supplied command env.
8
+ const gitEnv = (env = cleanEnv()) => ({...env,GIT_NO_REPLACE_OBJECTS:'1'});
9
+ export const git = (cwd,args,opts={}) => exec('git',['--no-replace-objects','-c','core.hooksPath=/dev/null','-c','protocol.ext.allow=never','-C',cwd,...args],{cwd,...opts,env:gitEnv(opts.env)});
10
+ export const baseLock = b => `${b.path}.okf-lock`;
11
+ export const journalPath = b => `${b.path}.okf-publication.json`;
12
+ export function validateBase(root, base) {
13
+ const files=tree(root,{git:base.kind==='git' && base.root==='.'}); const meta=metadata(files,base);
14
+ const result=JSON.parse(exec(process.execPath,[validator,root,'--strict','--json'],{acceptedStatus:[0,1]}));
15
+ if(result.errors.length || result.warnings.length) fail('E_VALIDATION', [...result.errors,...result.warnings].join('; '));
16
+ return {files,meta,digest:digest(files)};
17
+ }
18
+ function clone(base, dest) {
19
+ safePath(dest);
20
+ if(fs.existsSync(dest)) fail('E_PATH',`staging destination exists: ${dest}`);
21
+ fs.mkdirSync(dirname(dest),{recursive:true});
22
+ git(dirname(dest),['clone','--no-hardlinks','--no-checkout','--',base.repository,dest]);
23
+ git(dest,['fetch','origin',`refs/heads/${base.acceptedBranch}`]);
24
+ const head=git(dest,['rev-parse','FETCH_HEAD']);
25
+ git(dest,['checkout','--detach',head]);
26
+ // Reject a linked bundle even if Git happily checked the link out.
27
+ safePath(join(dest,base.root));
28
+ return head;
29
+ }
30
+ export function stageBase(base, dest) {
31
+ safePath(dest);
32
+ if(fs.existsSync(dest)) fail('E_PATH',`staging destination exists: ${dest}`);
33
+ const acceptedPath=base.kind==='directory'?base.path:(base.repository.startsWith('/')?resolve(base.repository,base.root):null);
34
+ if(acceptedPath && overlaps(acceptedPath,dest)) fail('E_PATH','stage overlaps accepted base');
35
+ if(base.kind==='git') {
36
+ const head=clone(base,dest); const root=join(dest,base.root);
37
+ const validated=validateBase(root,base);
38
+ verifyPublicationTree(base,dest,head,head,validated.files);
39
+ return { ...validated, head, root, checkout:dest };
40
+ }
41
+ noGit(base.path);
42
+ return withLock(baseLock(base),()=>{
43
+ if(fs.existsSync(journalPath(base))) fail('E_RECOVERY',`publication pending: ${journalPath(base)}; retry its recorded run before reading`);
44
+ const result=validateBase(base.path,base); materialize(dest,result.files);
45
+ return {...result,root:dest};
46
+ });
47
+ }
48
+ export function allowedChanges(before, after, meta, owned) {
49
+ const changed=[...new Set([...Object.keys(before),...Object.keys(after)])].filter(p=>before[p]!==after[p]);
50
+ if(changed.includes('okf-base.json')) fail('E_OWNER','harvest cannot change base identity/ownership');
51
+ const paths=owned.map(n=>meta.nodes[n]?.path || fail('E_OWNER',`unknown owned node ${n}`));
52
+ for(const p of changed) if(!['index.md','log.md'].includes(p) && !paths.some(n=>p.startsWith(n+'/'))) fail('E_OWNER',`unauthorized touched path: ${p}`);
53
+ // Base navigation can only gain/remove listings for this worker's nodes.
54
+ if(changed.includes('index.md')) {
55
+ const filter=value=>Buffer.from(value || '', 'base64').toString().split('\n').filter(l=>!paths.some(p=>l.includes(`](${p}/`) || l.includes(`](/${p}/`))).join('\n');
56
+ if(filter(before['index.md'])!==filter(after['index.md'])) fail('E_OWNER','base index edits must be navigation for owned nodes only');
57
+ }
58
+ if(changed.includes('log.md')) {
59
+ const old=Buffer.from(before['log.md']||'','base64').toString();
60
+ const next=Buffer.from(after['log.md']||'','base64').toString();
61
+ if(!next.includes(old)) fail('E_OWNER','base log is append/prepend-only, preserve history bytes');
62
+ }
63
+ return changed;
64
+ }
65
+ export function verifyGitScope(base, dest, baseline, { checkModes = true } = {}) {
66
+ const names=git(dest,['diff','--name-only','-z',baseline,'--']).split('\0').filter(Boolean);
67
+ // A restored working file can conceal a staged, unauthorized index entry.
68
+ names.push(...git(dest,['diff','--cached','--name-only','-z',baseline,'--']).split('\0').filter(Boolean));
69
+ names.push(...git(dest,['ls-files','--others','--exclude-standard','-z']).split('\0').filter(Boolean));
70
+ if(base.root!=='.' && names.some(p=>!p.startsWith(base.root+'/'))) fail('E_OWNER','Git worker touched files outside its knowledge base');
71
+ // --others without exclude-standard includes ignored additions as well.
72
+ // Unchanged code symlinks outside the base are not knowledge and need not be
73
+ // traversed. Bundle symlinks are rejected by validateBase/tree.
74
+ const extra=git(dest,['ls-files','--others','-z']).split('\0').filter(Boolean);
75
+ if(base.root!=='.' && extra.some(p=>!p.startsWith(base.root+'/'))) fail('E_OWNER','untracked/ignored file outside knowledge');
76
+ if(checkModes) {
77
+ // Harvest judgments authorize content, not executable-bit edits. Inspect
78
+ // the filesystem against frozen objects, not core.fileMode or index flags.
79
+ for(const [p,entry] of gitTreeEntries(dest,baseline)) {
80
+ if(!['100644','100755'].includes(entry.mode)) continue;
81
+ const path=join(dest,p);
82
+ let stat;try {stat=fs.lstatSync(path);} catch(e) {if(e.code==='ENOENT' || e.code==='ENOTDIR') continue;throw e;}
83
+ if(stat.isFile() && (stat.mode & 0o100 ? '100755':'100644')!==entry.mode) fail('E_OWNER',`harvest cannot change Git file mode: ${p}`);
84
+ }
85
+ }
86
+ verifyRemote(base,dest);
87
+ }
88
+ // Caller MUST hold the cooperative base lock. Intent is durable before journal
89
+ // installation; publishing is durable AFTER it and BEFORE the first accepted
90
+ // write. The journal is removed only after a durable accepted receipt. Thus an
91
+ // absent journal can cancel an intent, but never a write-authorized publication.
92
+ export function reconcileDirectoryIntent(base, receipt, persist) {
93
+ if(fs.existsSync(journalPath(base))) return;
94
+ if(receipt.status==='publication-intent') {
95
+ receipt.status='validated'; persist();
96
+ } else if(!['validated','staged','accepted','no-change'].includes(receipt.status)) {
97
+ fail('E_RECOVERY','directory publication journal missing after writes were authorized; inspect and restore custody before retrying or rejudging');
98
+ }
99
+ }
100
+ export function directoryPublish(base, proposal, receipt, persist, { afterWrite } = {}) {
101
+ noGit(base.path);
102
+ return withLock(baseLock(base),()=>{
103
+ const jp=journalPath(base);
104
+ reconcileDirectoryIntent(base,receipt,persist);
105
+ if(receipt.status==='accepted' && !fs.existsSync(jp)) return receipt;
106
+ const changed=[...new Set([...Object.keys(proposal.before),...Object.keys(proposal.after)])].filter(p=>proposal.before[p]!==proposal.after[p]);
107
+ // A mount boundary cannot support our cross-directory atomic rename.
108
+ // Detect it BEFORE journal intent or any deletion/replacement, rather than
109
+ // leaving a partially applied publication that can only fail with EXDEV.
110
+ const device=fs.statSync(baseLock(base)).dev;
111
+ for(const p of changed.filter(p=>proposal.after[p]!==undefined)) {
112
+ let parent=dirname(safePath(join(base.path,relPath(p))));
113
+ while(!fs.existsSync(parent)) parent=dirname(parent);
114
+ if(fs.statSync(parent).dev!==device) fail('E_PATH','directory publication targets and sibling lock must share a filesystem');
115
+ }
116
+ let journal;
117
+ if(fs.existsSync(jp)) {
118
+ journal=readJSON(jp);
119
+ if(journal.run!==proposal.run || journal.proposalHash!==hash(proposal)) fail('E_RECOVERY',`another publication must recover first: ${jp}`);
120
+ } else {
121
+ const current=validateBase(base.path,base);
122
+ if(current.digest!==digest(proposal.before)) fail('E_BASELINE','directory base changed; retain evidence and rejudge on a fresh run');
123
+ journal={version:1,run:proposal.run,proposalHash:hash(proposal),proposalFile:proposal.file,status:'publishing'};
124
+ receipt.status='publication-intent'; persist();
125
+ atomic(jp,JSON.stringify(journal,null,2)+'\n',{tempDir:baseLock(base)});
126
+ }
127
+ if(receipt.status==='accepted') {
128
+ // A death after receipt persistence needs only journal cleanup, never
129
+ // another accepted write (nor a replay over unexpected newer bytes).
130
+ const final=validateBase(base.path,base);
131
+ if(final.digest!==digest(proposal.after) || receipt.acceptedDigest!==final.digest) fail('E_CONFIRM','accepted directory receipt differs from journalled bytes');
132
+ fs.rmSync(jp); syncParent(jp); return receipt;
133
+ }
134
+ const current=tree(base.path);
135
+ for(const p of new Set([...Object.keys(current),...Object.keys(proposal.before),...Object.keys(proposal.after)])) {
136
+ if(current[p]!==proposal.before[p] && current[p]!==proposal.after[p]) fail('E_BASELINE',`unexpected bytes during publication recovery: ${p}`);
137
+ if(!changed.includes(p) && current[p]!==proposal.before[p]) fail('E_BASELINE',`unchanged file differs: ${p}`);
138
+ }
139
+ receipt.status='publishing'; persist();
140
+ let count=0;
141
+ for(const p of changed) {
142
+ const target=safePath(join(base.path,relPath(p)));
143
+ // A previous death may follow rename/unlink but precede its directory
144
+ // fsync. Confirm that metadata without replacing accepted bytes again.
145
+ if(current[p]===proposal.after[p]) {syncParent(target);continue;}
146
+ if(proposal.after[p]===undefined) { fs.rmSync(target,{force:true}); syncParent(target); }
147
+ // The lock owns scratch files as well as owner.json. A dead-owner unlock
148
+ // removes incomplete scratch; it never leaves temp files in the bundle.
149
+ else atomic(target,Buffer.from(proposal.after[p],'base64'),{tempDir:baseLock(base)});
150
+ afterWrite?.(++count); // fault injection is programmatic tests only, never an environment switch
151
+ }
152
+ const final=validateBase(base.path,base);
153
+ if(final.digest!==digest(proposal.after)) fail('E_CONFIRM','directory publication confirmation differs');
154
+ receipt.status='accepted'; receipt.acceptedDigest=final.digest; receipt.acceptedAt=new Date().toISOString(); persist();
155
+ fs.rmSync(jp); syncParent(jp);
156
+ return receipt;
157
+ });
158
+ }
159
+ function syncParent(p) { const fd=fs.openSync(dirname(p),'r'); try {fs.fsyncSync(fd);} finally {fs.closeSync(fd);} }
160
+ function prRows(base,branch,cwd,{identity,allBases=false}={}) {
161
+ const fields='number,url,state,headRefOid,baseRefName,headRefName,mergedAt,mergeCommit';
162
+ // Once observed, a PR is addressed by repository + number, never a mutable
163
+ // head/base filter. A missing/inaccessible identity is an error, not absence.
164
+ if(identity) {
165
+ if(!Number.isInteger(identity.number) || identity.number<1 || !/^https:\/\//.test(identity.url)) fail('E_RECOVERY','invalid recorded PR identity');
166
+ const pr=JSON.parse(exec('gh',['pr','view',String(identity.number),'--repo',base.pr.repository,'--json',fields],{cwd,env:gitEnv()}));
167
+ if(!pr || Array.isArray(pr) || typeof pr!=='object') fail('E_RECOVERY','known PR missing; reconcile custody before rejudging');
168
+ return [pr];
169
+ }
170
+ const raw=exec('gh',['pr','list','--repo',base.pr.repository,'--head',branch,...(allBases?[]:['--base',base.acceptedBranch]),'--state','all','--json',fields],{cwd,env:gitEnv()});
171
+ const rows=JSON.parse(raw); if(!Array.isArray(rows)) fail('E_PR','invalid gh PR list'); return rows;
172
+ }
173
+ function verifyRemote(base,cwd) {
174
+ for(const mode of [[],['--push']]) {
175
+ const urls=git(cwd,['remote','get-url',...mode,'--all','origin']).split('\n');
176
+ if(!urls.length || urls.some(url=>url!==base.repository)) fail('E_OWNER','worker changed frozen Git publication remote or effective push destination');
177
+ }
178
+ }
179
+ function changedPaths(before,after) {
180
+ return [...new Set([...Object.keys(before),...Object.keys(after)])].filter(p=>before[p]!==after[p]);
181
+ }
182
+ function gitTreeEntries(cwd,treeId) {
183
+ return new Map(git(cwd,['ls-tree','-r','-z',treeId]).split('\0').filter(Boolean).map(entry=>{
184
+ const tab=entry.indexOf('\t'),[mode,type,oid]=entry.slice(0,tab).split(' ');
185
+ return [entry.slice(tab+1),{mode,type,oid}];
186
+ }));
187
+ }
188
+ // Verify the REAL tree, not Git's working-tree view. Blob hashes include the
189
+ // exact bytes (no text trimming); Git filters/attributes and ignores cannot
190
+ // substitute something different from the validated proposal. Modes come only
191
+ // from the frozen baseline (new files are non-executable), never worker state.
192
+ function verifyPublicationTree(base,cwd,baseline,treeId,after,before=after) {
193
+ const names=git(cwd,['diff-tree','--no-commit-id','--no-renames','--name-only','-r','-z',baseline,treeId,'--']).split('\0').filter(Boolean);
194
+ if(base.root!=='.' && names.some(p=>!p.startsWith(base.root+'/'))) fail('E_OWNER','publication tree changes files outside its knowledge base');
195
+ const format=git(cwd,['rev-parse','--show-object-format']);
196
+ const prefix=base.root==='.'?'':base.root+'/';
197
+ const changed=new Set(changedPaths(before,after).map(p=>prefix+p));
198
+ if(names.some(p=>!changed.has(p))) fail('E_OWNER','publication tree changes paths outside the validated content changes');
199
+ const baselineEntries=gitTreeEntries(cwd,baseline);
200
+ const found=new Set();
201
+ for(const [path,{mode,type,oid}] of gitTreeEntries(cwd,treeId)) {
202
+ if(!path.startsWith(prefix)) continue;
203
+ const p=path.slice(prefix.length);
204
+ if(!Object.hasOwn(after,p) || type!=='blob' || !['100644','100755'].includes(mode)) fail('E_CONFIRM',`publication tree differs from validated proposal: ${p}`);
205
+ if(mode!==(baselineEntries.get(path)?.mode || '100644')) fail('E_OWNER',`publication tree changes frozen Git file mode: ${p}`);
206
+ const bytes=Buffer.from(after[p],'base64');
207
+ const expected=createHash(format).update(`blob ${bytes.length}\0`).update(bytes).digest('hex');
208
+ if(oid!==expected) fail('E_CONFIRM',`publication tree bytes differ from validated proposal: ${p}`);
209
+ found.add(p);
210
+ }
211
+ if(found.size!==Object.keys(after).length) fail('E_CONFIRM','publication tree omits validated proposal files (check Git ignores/attributes)');
212
+ }
213
+ function immutableCommit(cwd,oid) {
214
+ if(typeof oid!=='string' || !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(oid)) fail('E_CONFIRM','publication requires an immutable full commit OID');
215
+ if(git(cwd,['cat-file','-t',oid])!=='commit') fail('E_CONFIRM','publication OID must identify a commit object, not a peelable tag');
216
+ // Read object headers, not revision history (which can honor grafts).
217
+ const header=git(cwd,['cat-file','commit',oid]).split('\n\n')[0].split('\n');
218
+ return {tree:header.find(l=>l.startsWith('tree '))?.slice(5),parents:header.filter(l=>l.startsWith('parent ')).map(l=>l.slice(7))};
219
+ }
220
+ export function gitPublish(base, stage, proposal, receipt, persist, {beforePublish=()=>{},prIdentity=receipt.pr}={}) {
221
+ const cwd=stage.checkout, branch=`okf/${proposal.attempt || proposal.run}-${base.id}`;
222
+ verifyRemote(base,cwd);
223
+ const baseline=immutableCommit(cwd,stage.head);
224
+ verifyPublicationTree(base,cwd,baseline.tree,baseline.tree,proposal.before);
225
+ // Baseline must still be accepted. Never rebase model output without rejudging it.
226
+ git(cwd,['fetch','origin',`refs/heads/${base.acceptedBranch}`]);
227
+ const accepted=git(cwd,['rev-parse','FETCH_HEAD']);
228
+ if(accepted!==stage.head) {
229
+ // A known or uncertain previously created PR may be reconciled, but a
230
+ // committed/pushed proposal alone does not authorize a NEW stale-base PR.
231
+ const prior=receipt.commit?prRows(base,branch,cwd,{identity:prIdentity}).filter(p=>p.headRefOid===receipt.commit && p.headRefName===branch && p.baseRefName===base.acceptedBranch):[];
232
+ if(prior.length!==1) fail('E_BASELINE','accepted Git head changed before verified PR delivery; explicit rejudgment required');
233
+ }
234
+ if(!receipt.commit) {
235
+ // Publication consumes a frozen content proposal, also on migration/retry
236
+ // where materialization may have reset filesystem permissions. Its modes
237
+ // are reconstructed below, not authorized by the mutable worktree.
238
+ verifyGitScope(base,cwd,stage.head,{checkModes:false});
239
+ if(digest(tree(stage.root,{git:base.root==='.'}))!==digest(proposal.after)) fail('E_BASELINE','staged proposal changed after validation');
240
+ beforePublish();
241
+ receipt.status='commit-intent'; receipt.branch=branch; persist();
242
+ // Never trust the model's index. Start a private publication index from
243
+ // the frozen accepted commit, and leave the worker's own index intact.
244
+ const scratch=fs.mkdtempSync(join(dirname(proposal.file),'git-index-'));
245
+ let treeId;
246
+ try {
247
+ const env={...cleanEnv(),GIT_INDEX_FILE:join(scratch,'index')};
248
+ git(cwd,['read-tree',baseline.tree],{env});
249
+ const prefix=base.root==='.'?'':base.root+'/';
250
+ const entries=gitTreeEntries(cwd,baseline.tree);
251
+ // Recompute the delta from validated bytes, never trust proposal.changed
252
+ // or stage an entire directory (which also collects mode-only edits).
253
+ for(const p of changedPaths(proposal.before,proposal.after)) {
254
+ const path=prefix+relPath(p);
255
+ // An ignored addition can exit 1. The exact tree check below must still
256
+ // reject its omission; no ignore/filter failure becomes a partial PR.
257
+ git(cwd,['--literal-pathspecs','add','--all','--chmod=-x','--',path],{env,acceptedStatus:[0,1]});
258
+ if(Object.hasOwn(proposal.after,p) && entries.get(path)?.mode==='100755') git(cwd,['--literal-pathspecs','update-index','--chmod=+x','--',path],{env});
259
+ }
260
+ treeId=git(cwd,['write-tree'],{env});
261
+ verifyPublicationTree(base,cwd,baseline.tree,treeId,proposal.after,proposal.before);
262
+ } finally {fs.rmSync(scratch,{recursive:true,force:true});}
263
+ // Deterministic commit-tree permits replay after process death between the
264
+ // real Git write and receipt persistence; no guessed commit or fake repository.
265
+ const stamp=proposal.created;
266
+ const env={...cleanEnv(),GIT_AUTHOR_NAME:'OKF harvest',GIT_AUTHOR_EMAIL:'okf@localhost',GIT_COMMITTER_NAME:'OKF harvest',GIT_COMMITTER_EMAIL:'okf@localhost',GIT_AUTHOR_DATE:stamp,GIT_COMMITTER_DATE:stamp};
267
+ receipt.commit=git(cwd,['commit-tree',treeId,'-p',stage.head,'-m',`memory-harvest: ${proposal.run}`],{env});
268
+ receipt.status='committed'; persist();
269
+ }
270
+ const publication=immutableCommit(cwd,receipt.commit);
271
+ if(publication.parents.length!==1 || publication.parents[0]!==stage.head) fail('E_CONFIRM','publication commit must have exactly the frozen baseline as its parent');
272
+ verifyPublicationTree(base,cwd,baseline.tree,publication.tree,proposal.after,proposal.before);
273
+ const remoteTip=()=>git(cwd,['ls-remote','--heads','origin',`refs/heads/${branch}`]).split(/\s/)[0] || null;
274
+ let tip=remoteTip();
275
+ if(tip && tip!==receipt.commit) fail('E_PR','publication branch has unexpected commit; never force push');
276
+ if(tip!==receipt.commit) {
277
+ beforePublish();
278
+ receipt.status='push-intent'; persist();
279
+ try { verifyRemote(base,cwd);git(cwd,['push','--no-follow-tags','--recurse-submodules=no','origin',`${receipt.commit}:refs/heads/${branch}`]); }
280
+ catch(e) { receipt.status='push-unknown'; receipt.error=e.message; persist(); throw e; }
281
+ tip=remoteTip(); if(tip!==receipt.commit) {receipt.status='push-unknown';persist();fail('E_CONFIRM','pushed head not confirmed');}
282
+ }
283
+ receipt.status='pushed'; persist();
284
+ let rows;
285
+ try {rows=prRows(base,branch,cwd,{identity:prIdentity});} catch(e) {receipt.status='pr-unknown';receipt.error=e.message;persist();throw e;}
286
+ if(!rows.length) {
287
+ beforePublish();
288
+ receipt.status='pr-intent'; persist();
289
+ try { exec('gh',['pr','create','--repo',base.pr.repository,'--head',branch,'--base',base.acceptedBranch,'--title',`memory-harvest: ${proposal.run}`,'--body',`Knowledge-only proposal from durable OKF input ${proposal.run}. Review provenance and promotion judgment.`],{cwd,env:gitEnv()}); }
290
+ catch(e) {receipt.status='pr-unknown';receipt.error=e.message;persist();throw e;}
291
+ rows=prRows(base,branch,cwd);
292
+ }
293
+ const matching=rows.filter(p=>p.headRefOid===receipt.commit && p.headRefName===branch && p.baseRefName===base.acceptedBranch && Number.isInteger(p.number) && /^https:\/\//.test(p.url) && (!prIdentity || (p.number===prIdentity.number && p.url===prIdentity.url)));
294
+ if(matching.length!==1 || rows.length!==1) {receipt.status='pr-unknown';persist();fail('E_PR','actual PR identity/head/base could not be uniquely verified');}
295
+ const pr=matching[0];receipt.pr=pr;receipt.status='delivered';receipt.deliveredAt ||= new Date().toISOString();persist();
296
+ if(pr.state==='CLOSED' && !pr.mergedAt) {receipt.status='rejected';persist();fail('E_PR','PR closed without merge; retained proposal requires operator review');}
297
+ if(pr.mergedAt) {
298
+ git(cwd,['fetch','origin',`refs/heads/${base.acceptedBranch}`]);
299
+ const merge=pr.mergeCommit?.oid;
300
+ if(!merge) fail('E_CONFIRM','merged PR lacks merge commit');
301
+ git(cwd,['merge-base','--is-ancestor',merge,'FETCH_HEAD']);
302
+ receipt.status='accepted';receipt.acceptedCommit=git(cwd,['rev-parse','FETCH_HEAD']);receipt.acceptedAt=new Date().toISOString();persist();
303
+ }
304
+ return receipt;
305
+ }
306
+
307
+ export function recoveryStage(base, stage, proposal, dest) {
308
+ clone(base,dest);
309
+ git(dest,['checkout','--detach',stage.head]);
310
+ const root=join(dest,base.root);
311
+ for(const p of Object.keys(proposal.before)) if(!(p in proposal.after)) fs.rmSync(safePath(join(root,p)),{force:true});
312
+ materialize(root,proposal.after);validateBase(root,base);
313
+ return {...stage,root,checkout:dest};
314
+ }
315
+
316
+ // Remote-read-only recovery gate. --repo makes this independent of any deleted
317
+ // worker checkout. The caller durably saves first observations separately from
318
+ // historical receipts, even if a later gate prevents recovery from completing.
319
+ export function gitRecoveryState(base,receipt,cwd,{identity=receipt.pr,onObserve=()=>{}}={}) {
320
+ if(['accepted','no-change'].includes(receipt.status)) return 'settled';
321
+ if(!receipt.branch) return 'unresolved';
322
+ const rows=prRows(base,receipt.branch,cwd,{identity,allBases:true});
323
+ if(!rows.length) {
324
+ if(identity || ['delivered','rejected'].includes(receipt.status)) fail('E_RECOVERY','known PR missing; reconcile custody before rejudging');
325
+ return 'unresolved';
326
+ }
327
+ const pr=rows[0];
328
+ if(rows.length!==1 || pr.headRefOid!==receipt.commit || pr.headRefName!==receipt.branch || pr.baseRefName!==base.acceptedBranch || !Number.isInteger(pr.number) || pr.number<1 || !/^https:\/\//.test(pr.url) || [identity,receipt.pr].some(known=>known && (pr.number!==known.number || pr.url!==known.url)) || !['OPEN','CLOSED','MERGED'].includes(pr.state) || (pr.state==='MERGED' && !pr.mergedAt)) fail('E_RECOVERY','actual PR identity/head/base could not be uniquely verified for recovery');
329
+ onObserve(pr);
330
+ return pr.state==='CLOSED' && !pr.mergedAt?'unresolved':'settled';
331
+ }