@petercjl/codex-own 0.3.0 → 0.4.0

package/README.md

@@ -8,6 +8,14 @@
 
 日常应用内更新使用随 App 交付的更新组件，无需手动更新全局 npm 管理工具。API Key、配置、对话和语言偏好会保留。断网时当前 App 保持可用；下载失败可以重试，已校验分片会复用；已准备好的更新会在下次打开时恢复。旧版本首次升级到带更新组件的版本，以及故障恢复，可以使用下文的终端更新命令。
 
+## 增量下载
+
+从 0.4.0 起，更新前会扫描已安装的 codex-own 程序，复用与新版相同的内容块，只从 npm 下载缺少的块。支持跳过版本；首次安装仍下载完整资源。进度会显示“复用多少块、需要下载多少 MiB、全量多少 MiB”，结果中的 `transfer` 保留这些统计。少量代码改动通常只影响部分块；上游引擎、大型二进制或签名资源变化时，下载量会相应增加。
+
+0.3.x 首次迁移：先退出 codex-own，使用下文对应系统的终端命令更新 npm 管理工具，再执行桌面更新。升级后的管理工具即可从现有程序复用内容。之后直接使用应用内“更新”菜单，随 App 交付的更新组件会自动执行相同的增量下载。
+
+本地扫描只读取程序目录，密钥、配置和对话继续保存在独立数据目录。下载结束后在本地重建完整归档、核对 SHA256，再备份并切换程序。请继续预留完整安装和备份的磁盘空间。
+
 ## 在 Mac 上通过 Terminal 安装和更新
 
 适用于 Apple 芯片 Mac（ARM64）。从 Node.js 官网安装 LTS 版本（要求 Node.js 22 或以上，包含 npm），重新打开 Terminal。建议预留 10 GB 空间，用于下载、解压和备份。
@@ -82,7 +90,7 @@ $codexOwn = Join-Path (npm.cmd prefix --global) "codex-own.cmd"
 & $codexOwn install --yes
 ```
 
-管理工具和桌面资源全部从 registry.npmjs.org 下载。Windows 桌面下载量约 878 MiB，最多并行下载 4 个分片，安装时显示分片编号、下载百分比和已接收大小，然后显示校验、解压、安装。看到“安装完成”和 ok: true 才表示完成。
+管理工具和桌面资源全部从 registry.npmjs.org 下载。Windows 桌面资源约 1 GB，具体大小随版本变化。最多并行下载 4 个分片，安装时显示分片编号、下载百分比和已接收大小，然后显示校验、解压、安装。看到“安装完成”和 ok: true 才表示完成。
 
 若已有手动安装的 codex-own，先退出应用，将第二步的安装命令替换为以下接管命令；原安装会被备份。
 

package/capabilities.json

@@ -25,12 +25,25 @@
     "runtimeHosts": [
       "registry.npmjs.org"
     ],
-    "desktopAssets": "content-addressed npm payload prereleases; install latest manager only",
-    "cache": "verified platform parts retained for retries",
+    "desktopAssets": "canonical-tar-cdc-v1; content-defined raw program blocks compressed and distributed as npm payload prereleases",
+    "cache": "SHA256-verified blocks; seed matching blocks from managed installed program; download missing content only",
     "progress": "stderr; final result JSON on stdout",
     "idleTimeoutSeconds": 30,
     "downloadAttempts": 3,
-    "parallelDownloads": 4
+    "parallelDownloads": 4,
+    "incremental": {
+      "format": "canonical-tar-cdc-v1",
+      "rawBlockMinMiB": 16,
+      "rawBlockMaxMiB": 64,
+      "baseVersionRequired": false,
+      "firstInstall": "all required blocks",
+      "migration": "upgrade npm manager once for desktop builds older than 0.4.0",
+      "statistics": [
+        "reusedBlocks",
+        "downloadBytes",
+        "fullDownloadBytes"
+      ]
+    }
   },
   "desktopUpdater": {
     "source": "https://registry.npmjs.org/@petercjl%2fcodex-own/latest",

package/lib/app-update.mjs

@@ -2,6 +2,7 @@ import fs from 'node:fs';
 import path from 'node:path';
 import {latest,manifestFor,validateManifest,paths,state,locked,rejectLinks,exclusive,digestFile,tarExecutable,install,fail,running,launch} from './manager.mjs';
 import {assembleNpmAsset} from './npm-assets.mjs';
+import {reuseInstalledBlocks} from './content-blocks.mjs';
 
 // Desktop release versions use the stable npm channel, independently of upstream's app version.
 export function isNewer(candidate,current) {
@@ -27,12 +28,14 @@ export async function prepareUpdate(current,{p=paths(),resolve=latest,manifest=m
     const dir=fs.mkdtempSync(path.join(base,'update-')),archive=path.join(dir,'desktop.tar.gz');
     const cache=path.join(p.manager,'npm-assets');rejectLinks(cache);
     try{
-      await assemble(asset,archive,cache,tarExecutable(),{report,signal});
+      // The app may stay open: only read program resources, never user data.
+      if(p.target){rejectLinks(p.target);await reuseInstalledBlocks(asset,p.target,cache,{report,signal});}
+      const transfer=await assemble(asset,archive,cache,tarExecutable(),{report,signal});
       // Verify even when a download adapter is supplied.
       if(fs.statSync(archive).size!==asset.size||await digestFile(archive)!==asset.sha256)fail('CHECKSUM_MISMATCH','Prepared update checksum mismatch');
       const receipt=path.join(dir,'receipt.json');
-      exclusive(receipt,{schema:1,current,version:m.version,platform:p.platform,manifest:m});
-      return {...checked,receipt};
+      exclusive(receipt,{schema:1,current,version:m.version,platform:p.platform,manifest:m,...(transfer?{transfer}:{})});
+      return {...checked,receipt,...(transfer?{transfer}:{})};
     }catch(e){fs.rmSync(dir,{recursive:true,force:true});throw e;}
   });
 }

package/lib/content-blocks.mjs

@@ -0,0 +1,121 @@
+// Versioned, deterministic transport shared by build, local reuse and release.
+// npm compresses each raw block; compressing the whole app first destroys reuse.
+import fs from 'node:fs';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import {pipeline} from 'node:stream/promises';
+
+export const TRANSPORT = 'canonical-tar-cdc-v1';
+export const BLOCKS = Object.freeze({min:16*1048576, mask:0x1ffffff, max:64*1048576});
+const sha = b => crypto.createHash('sha256').update(b).digest('hex');
+const abort = signal => { if(signal?.aborted) throw Object.assign(Error('Update cancelled'), {code:'CANCELLED'}); };
+const gear = Array.from({length:256},(_,i)=>crypto.createHash('sha256').update(`codex-own-cdc-v1:${i}`).digest().readUInt32LE(0));
+const fileBoundary=()=>Object.assign(Buffer.alloc(0),{canonicalFileBoundary:true});
+
+function header(name, size, type, mode, link='') {
+  const b=Buffer.alloc(512);
+  const text=(s,at,len)=>{if(Buffer.byteLength(s)>len)throw Error('Tar field overflow');b.write(s,at,len,'utf8');};
+  const oct=(n,at,len)=>text(n.toString(8).padStart(len-1,'0')+'\0',at,len);
+  text(name,0,100);oct(mode,100,8);oct(0,108,8);oct(0,116,8);
+  oct(size,124,12);oct(0,136,12);b.fill(32,148,156);text(type,156,1);
+  text(link,157,100);text('ustar\0',257,6);text('00',263,2);
+  const sum=b.reduce((a,v)=>a+v,0);text(sum.toString(8).padStart(6,'0')+'\0 ',148,8);
+  return b;
+}
+function paxRecord(key,value) {
+  const body=` ${key}=${value}\n`;let n=Buffer.byteLength(body)+1;
+  while(String(n).length+Buffer.byteLength(body)!==n)n=String(n).length+Buffer.byteLength(body);
+  return Buffer.from(String(n)+body);
+}
+function* entryHeaders(name,size,type,mode,link='') {
+  const records=[];
+  if(Buffer.byteLength(name)>100)records.push(paxRecord('path',name));
+  if(Buffer.byteLength(link)>100)records.push(paxRecord('linkpath',link));
+  if(records.length){
+    const data=Buffer.concat(records),id=sha(Buffer.from(name));
+    yield header('PaxHeaders/'+id,data.length,'x',0o644);yield data;
+    if(data.length%512)yield Buffer.alloc(512-data.length%512);
+    name='entry-'+id;
+  }
+  yield header(name,size,type,mode,Buffer.byteLength(link)>100?'':link);
+}
+
+// Stable order, permissions and timestamps across CI and an installed app.
+// Links are recorded, never traversed. No host metadata or user data is bundled.
+export async function* canonicalTar(root,entry,{signal}={}) {
+  if(!['app','codex-own.app'].includes(entry))throw Error('Unexpected application entry');
+  root=path.resolve(root);
+  if(!fs.lstatSync(root).isDirectory()||fs.lstatSync(root).isSymbolicLink())throw Error('Expected a real application directory');
+  async function* walk(file,name){
+    abort(signal);const st=fs.lstatSync(file);
+    if(st.isSymbolicLink()){
+      const link=fs.readlinkSync(file),resolved=path.resolve(path.dirname(file),link);
+      if(path.isAbsolute(link)||(resolved!==root&&!resolved.startsWith(root+path.sep)))throw Object.assign(Error('Escaping application symlink'),{code:'UNSAFE_PATH'});
+      yield* entryHeaders(name,0,'2',0o777,link);
+    }else if(st.isDirectory()){
+      yield* entryHeaders(name+'/',0,'5',0o755);
+      for(const n of fs.readdirSync(file).sort())yield* walk(path.join(file,n),name+'/'+n);
+    }else if(st.isFile()){
+      // A changed file must not shift the block grid of the next large binary.
+      if(st.size>=BLOCKS.min)yield fileBoundary();
+      const mode=entry==='codex-own.app'&&(st.mode&0o111)?0o755:0o644;
+      yield* entryHeaders(name,st.size,'0',mode);let size=0;
+      for await(const b of fs.createReadStream(file)){abort(signal);size+=b.length;yield b;}
+      if(size!==st.size)throw Object.assign(Error('Application changed during local scan'),{code:'LOCAL_APP_CHANGED'});
+      if(size%512)yield Buffer.alloc(512-size%512);
+      if(st.size>=BLOCKS.min)yield fileBoundary();
+    }else throw Object.assign(Error('Unsupported application file'),{code:'UNSAFE_PATH'});
+  }
+  yield* walk(root,entry);yield Buffer.alloc(1024);
+}
+
+// Boundaries depend on content, so insertion in an asar does not shift all blocks.
+export async function* contentBlocks(stream,{min=BLOCKS.min,mask=BLOCKS.mask,max=BLOCKS.max,signal}={}) {
+  if(!Number.isInteger(min)||min<1||!Number.isInteger(max)||max<min||max>128*1048576||!Number.isInteger(mask)||mask<1)throw Error('Invalid block parameters');
+  let pieces=[],size=0,h=0;
+  for await(const buffer of stream){
+    abort(signal);
+    if(buffer.canonicalFileBoundary){if(size)yield Buffer.concat(pieces,size);pieces=[];size=0;h=0;continue;}
+    let start=0;
+    for(let i=0;i<buffer.length;i++){
+      h=((h<<1)+gear[buffer[i]])>>>0;size++;
+      if(size>=min&&((h&mask)===0||size>=max)){
+        pieces.push(buffer.subarray(start,i+1));yield Buffer.concat(pieces,size);
+        pieces=[];size=0;h=0;start=i+1;
+      }
+    }
+    if(start<buffer.length)pieces.push(buffer.subarray(start));
+  }
+  if(size)yield Buffer.concat(pieces,size);
+}
+
+export async function writeCanonicalArchive(root,entry,dest,{signal}={}) {
+  const blocks=[],digest=crypto.createHash('sha256');let size=0;
+  async function* bytes(){
+    for await(const b of contentBlocks(canonicalTar(root,entry,{signal}),{signal})){
+      blocks.push({sha256:sha(b),size:b.length,offset:size});size+=b.length;digest.update(b);yield b;
+    }
+  }
+  await pipeline(bytes(),fs.createWriteStream(dest,{flags:'wx',mode:0o600}),...(signal?[{signal}]:[]));
+  return {transport:TRANSPORT,size,sha256:digest.digest('hex'),blocks};
+}
+
+export async function reuseInstalledBlocks(asset,root,cache,{signal,report=()=>{}}={}) {
+  if(asset.transport!==TRANSPORT||!root||!fs.existsSync(root))return {seededBlocks:0,seededBytes:0};
+  fs.mkdirSync(cache,{recursive:true,mode:0o700});
+  const wanted=new Map(asset.parts.map(p=>[p.sha256,p.size]));
+  let seededBlocks=0,seededBytes=0,scanned=0,last=0;
+  report('正在扫描已安装程序，匹配可复用的内容块');
+  for await(const b of contentBlocks(canonicalTar(root,asset.entry,{signal}),{signal})){
+    scanned+=b.length;const hash=sha(b);
+    if(wanted.get(hash)===b.length){
+      const file=path.join(cache,hash+'.bin');
+      // Never replace an existing cache entry here. Assembly verifies it again.
+      if(!fs.existsSync(file))fs.writeFileSync(file,b,{flag:'wx',mode:0o600});
+      wanted.delete(hash);seededBlocks++;seededBytes+=b.length;
+    }
+    if(Date.now()-last>2000){report(`本地扫描 ${Math.round(scanned/1048576)} MiB · 匹配 ${seededBlocks} 个内容块`);last=Date.now();}
+  }
+  report(`本地匹配完成：可复用 ${seededBlocks} 个内容块`);
+  return {seededBlocks,seededBytes};
+}

package/lib/manager.mjs

@@ -6,6 +6,7 @@ import crypto from 'node:crypto';
 import {execFileSync, spawn} from 'node:child_process';
 import {networkFetch, verifiedDownload, progress} from './network.mjs';
 import {validateParts, assembleNpmAsset} from './npm-assets.mjs';
+import {reuseInstalledBlocks} from './content-blocks.mjs';
 
 export const packageRoot = path.resolve(import.meta.dirname, '..');
 export const pkg = JSON.parse(fs.readFileSync(path.join(packageRoot,'package.json'),'utf8'));
@@ -51,10 +52,13 @@ export function unpack(archive,dest) {
   function walk(p){for(const x of fs.readdirSync(p,{withFileTypes:true})){const f=path.join(p,x.name);if(x.isSymbolicLink()){const resolved=path.resolve(path.dirname(f),fs.readlinkSync(f));if(!resolved.startsWith(dest+path.sep))fail('UNSAFE_ARCHIVE','Escaping symlink');}else if(x.isDirectory())walk(f);}}
   walk(dest);
 }
+export function macOwnProcessIds(lines) {
+  return lines.filter(l=>l.includes('codex-own.app/Contents/')&&!l.includes(' ps -')&&!l.includes('/Contents/Resources/cua_node/')&&!/\/Contents\/MacOS\/WinAppHelper(?:\s|$)/.test(l)).map(l=>l.trim().split(/\s+/)[0]);
+}
 export function running(p=paths()) {
   if(process.platform==='darwin'){
     const lines=execFileSync('ps',['-axo','pid=,command='],{encoding:'utf8'}).split('\n');
-    return lines.filter(l=>l.includes('codex-own.app/Contents/')&&!l.includes(' ps -')&&!l.includes('/Contents/Resources/cua_node/')).map(l=>l.trim().split(/\s+/)[0]);
+    return macOwnProcessIds(lines);
   }
   const out=execFileSync('powershell.exe',['-NoProfile','-Command',"@(Get-Process -Name 'codex-own' -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Id) -join ','"],{encoding:'utf8'}).trim();return out?out.split(','):[];
 }
@@ -102,9 +106,13 @@ export async function install({version,adopt=false,archive,manifest,expectedCurr
     if(expectedCurrent!==undefined&&state(p)?.version!==expectedCurrent)fail('INSTALLATION_CHANGED','Installation changed after the update was prepared; check again');
     if(state(p)?.version===m.version&&!archive)return {ok:true,unchanged:true,version:m.version};
     if(running(p).length)fail('APP_RUNNING','Quit codex-own before install/update');
-    const stage=fs.mkdtempSync(path.join(p.manager,'stage-'));let f=archive;
+    const stage=fs.mkdtempSync(path.join(p.manager,'stage-'));let f=archive,transfer;
     if(f){if(fs.statSync(f).size!==a.size||await digestFile(f)!==a.sha256)fail('CHECKSUM_MISMATCH','Local archive checksum mismatch');}
-    else{if(m.schema!==2)fail('NPM_ASSETS_REQUIRED','Update @petercjl/codex-own to the latest npm-only release.');f=path.join(stage,'app.tar.gz');const cache=path.join(p.manager,'npm-assets');rejectLinks(cache);await assembleNpmAsset(a,f,cache,tarExecutable());}
+    else{if(m.schema!==2)fail('NPM_ASSETS_REQUIRED','Update @petercjl/codex-own to the latest npm-only release.');f=path.join(stage,'app.tar');const cache=path.join(p.manager,'npm-assets');rejectLinks(cache);
+      if(state(p)?.identity==='local.codex-own'&&fs.existsSync(p.target)&&appIdentity(p.target)==='local.codex-own'){
+        rejectLinks(p.target);await reuseInstalledBlocks(a,p.target,cache,{report:progress});
+      }
+      transfer=await assembleNpmAsset(a,f,cache,tarExecutable());}
     progress('校验通过，正在解压桌面程序');
     const content=path.join(stage,'unpacked');unpack(f,content);const source=path.join(content,a.entry);
     if(process.platform==='darwin')execFileSync('codesign',['--verify','--deep','--strict',source],{stdio:'pipe'});
@@ -112,7 +120,7 @@ export async function install({version,adopt=false,archive,manifest,expectedCurr
     const result=switchApp({p,source,version:m.version,adopt});
     if(process.platform==='darwin')execFileSync('/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Support/lsregister',['-f',p.target],{stdio:'pipe'});
     else shortcut(p);
-    fs.rmSync(stage,{recursive:true,force:true});progress('安装完成');return {...result,downloadSource:archive?'local-archive':'registry.npmjs.org'};
+    fs.rmSync(stage,{recursive:true,force:true});progress('安装完成');return {...result,downloadSource:archive?'local-archive':'registry.npmjs.org',...(transfer?{transfer}:{})};
   });
 }
 export function shortcut(p) {

package/lib/npm-assets.mjs

@@ -3,12 +3,14 @@ import path from 'node:path';
 import {execFileSync} from 'node:child_process';
 import {pipeline} from 'node:stream/promises';
 import {fileHash, verifiedDownload, progress} from './network.mjs';
+import {TRANSPORT} from './content-blocks.mjs';
 
 export const payloadVersion = sha => `0.0.0-payload.h${sha}`;
 export const payloadURL = sha => `https://registry.npmjs.org/@petercjl/codex-own/-/codex-own-${payloadVersion(sha)}.tgz`;
 export function validateParts(asset) {
   const fail = () => { throw Object.assign(new Error('Invalid npm asset manifest'), {code:'INVALID_MANIFEST'}); };
-  if (!Array.isArray(asset.parts) || asset.parts.length < 1 || asset.parts.length > 64) fail();
+  if (asset.transport !== undefined && asset.transport !== TRANSPORT) fail();
+  if (!Array.isArray(asset.parts) || asset.parts.length < 1 || asset.parts.length > 1024) fail();
   let size = 0;
   for (const p of asset.parts) {
     if (!/^[a-f0-9]{64}$/.test(p.sha256) || !/^[a-f0-9]{64}$/.test(p.archiveSha256)
@@ -32,11 +34,22 @@ export async function assembleNpmAsset(asset, dest, cache, tar, {download = veri
   const cancel=()=>controller.abort();signal?.addEventListener('abort',cancel,{once:true});
   if(signal?.aborted)controller.abort();
   try{
+  const cached = new Set();
+  for(const part of asset.parts){
+    const file=path.join(cache,part.sha256+'.bin');safeCache(file);
+    if(fs.existsSync(file)&&fs.statSync(file).size===part.size&&await fileHash(file)===part.sha256)cached.add(part.sha256);
+  }
+  const uniqueParts=[...new Map(asset.parts.map(part=>[part.sha256,part])).values()];
+  const stats={transport:asset.transport||'legacy',totalBlocks:asset.parts.length,reusedBlocks:asset.parts.filter(p=>cached.has(p.sha256)).length,
+    downloadBlocks:uniqueParts.filter(p=>!cached.has(p.sha256)).length,
+    downloadBytes:uniqueParts.filter(p=>!cached.has(p.sha256)).reduce((n,p)=>n+p.archiveSize,0),
+    fullDownloadBytes:uniqueParts.reduce((n,p)=>n+p.archiveSize,0)};
+  report(`更新资源：复用 ${stats.reusedBlocks}/${stats.totalBlocks} 块 · 需下载 ${(stats.downloadBytes/1048576).toFixed(1)} MiB / 全量 ${(stats.fullDownloadBytes/1048576).toFixed(1)} MiB`);
   async function fetchPart(part, i) {
     report(`npm 资源分片 ${i+1}/${asset.parts.length}`);
     const file = path.join(cache, `${part.sha256}.bin`);
     safeCache(file);
-    if (fs.existsSync(file) && fs.statSync(file).size === part.size && await fileHash(file) === part.sha256) {
+    if (cached.has(part.sha256)) {
       report(`[${i+1}/${asset.parts.length}] 复用已校验分片`); return file;
     }
     if (fs.existsSync(file)) fs.unlinkSync(file);
@@ -66,11 +79,12 @@ export async function assembleNpmAsset(asset, dest, cache, tar, {download = veri
   if(failure)throw failure;
   if(controller.signal.aborted)throw Object.assign(Error('Download cancelled'),{code:'CANCELLED'});
   const files=asset.parts.map(part=>completed.get(part.sha256));
-  report('合并并校验完整桌面安装包');
+  report('在本地重建并校验完整桌面安装包');
   const output = fs.createWriteStream(dest, {flags:'wx',mode:0o600});
   async function* chunks() { for (const f of files) yield* fs.createReadStream(f); }
   await pipeline(chunks(), output,{signal:controller.signal});
   if (fs.statSync(dest).size !== asset.size || await fileHash(dest) !== asset.sha256)
     throw Object.assign(new Error('Assembled desktop checksum mismatch'), {code:'CHECKSUM_MISMATCH'});
+  return stats;
   }finally{signal?.removeEventListener('abort',cancel);}
 }