@feng0531/dsh-vcs 0.6.1 → 0.7.0

package/index.mjs

@@ -40,7 +40,7 @@ const REQUEST_FIELDS = {
   'vcs/branch-switch-prepare': ['sessionId', 'repositoryId', 'branch'],
   'vcs/branch-switch-confirm': ['sessionId', 'repositoryId', 'token'],
   'vcs/stash-list': ['sessionId', 'repositoryId'],
-  'vcs/grep': ['sessionId', 'repositoryId', 'commit', 'query'],
+  'vcs/grep': ['sessionId', 'repositoryId', 'commit', 'query', 'cursor', 'limit'],
   'vcs/status': ['sessionId', 'repositoryId', 'mode'],
   'vcs/compare': ['sessionId', 'repositoryId', 'mode', 'id', 'large'],
   'vcs/conflict-stages': ['sessionId', 'repositoryId', 'mode', 'id'],
@@ -55,7 +55,7 @@ const REQUEST_FIELDS = {
   'vcs/tree': ['sessionId', 'repositoryId', 'commit'],
   'vcs/tree-page': ['sessionId', 'repositoryId', 'commit', 'path', 'offset', 'limit', 'query'],
   'vcs/tree-file': ['sessionId', 'repositoryId', 'commit', 'path'],
-  'vcs/tree-segment': ['sessionId', 'repositoryId', 'commit', 'path', 'offset'],
+  'vcs/tree-segment': ['sessionId', 'repositoryId', 'commit', 'path', 'offset', 'line'],
   'vcs/tree-history': ['sessionId', 'repositoryId', 'commit', 'path', 'offset', 'limit', 'follow', 'since', 'until'],
   'vcs/commit-image': ['sessionId', 'repositoryId', 'commit', 'parentIndex', 'id', 'side'],
   'vcs/workspace-image': ['sessionId', 'repositoryId', 'mode', 'id', 'side'],
@@ -135,6 +135,8 @@ function validateContentSearch(endpoint, payload) {
   if (endpoint !== 'vcs/grep') return;
   if (typeof payload.query !== 'string' || !payload.query || payload.query.length > 1024 || /[\0\r\n]/.test(payload.query)) throw invalid('Invalid content search query.');
   if (payload.commit !== undefined && (typeof payload.commit !== 'string' || !isObjectId(payload.commit))) throw invalid('Invalid content search commit.');
+  if (payload.cursor !== undefined && (typeof payload.cursor !== 'string' || !payload.cursor || payload.cursor.length > 4096 || !payload.commit)) throw invalid('Invalid content search cursor.');
+  if (payload.limit !== undefined && (!Number.isInteger(payload.limit) || payload.limit < 1 || payload.limit > 100)) throw invalid('Invalid content search page size.');
 }
 
 function validateRenameFollow(endpoint, payload) {
@@ -178,6 +180,7 @@ function validateRepositoryPayload(endpoint, payload) {
   validateCommitSelection(endpoint, payload);
   validateLargeComparison(endpoint, payload);
   if (endpoint === 'vcs/tree-segment' && payload.offset !== undefined && (!Number.isInteger(payload.offset) || payload.offset < 0 || payload.offset > 16777216)) throw invalid('Invalid segment offset.');
+  if (endpoint === 'vcs/tree-segment' && payload.line !== undefined && (!Number.isSafeInteger(payload.line) || payload.line < 1 || payload.line > 16777217 || payload.offset !== undefined)) throw invalid('Invalid segment target line.');
   validateBlameWindow(endpoint, payload);
   validateContentSearch(endpoint, payload);
   validateRenameFollow(endpoint, payload);
@@ -215,7 +218,7 @@ async function readRepository(api, endpoint, repository, payload, {cwd, mode, si
         ...optionalField('since', payload.since), ...optionalField('until', payload.until), signal,
       });
     case 'vcs/tree-segment':
-      return api.getHistoricalSegment({...repository}, {commit: payload.commit, path: payload.path, offset: payload.offset ?? 0, signal});
+      return api.getHistoricalSegment({...repository}, {commit: payload.commit, path: payload.path, ...optionalField('offset', payload.offset), ...optionalField('line', payload.line), signal});
     case 'vcs/workspace-image':
       return api.getWorkspaceImage({...repository}, {mode, id: payload.id, side: payload.side ?? 'right', signal});
     case 'vcs/revision-image':
@@ -225,7 +228,7 @@ async function readRepository(api, endpoint, repository, payload, {cwd, mode, si
     case 'vcs/stash-list':
       return api.listStashes({...repository}, {signal});
     case 'vcs/grep':
-      return api.searchRepositoryContent({...repository}, {commit: payload.commit, query: payload.query, signal});
+      return api.searchRepositoryContent({...repository}, {commit: payload.commit, query: payload.query, ...optionalField('cursor', payload.cursor), ...optionalField('limit', payload.limit), signal});
     case 'vcs/commit-image':
       return api.getCommitImage({...repository}, {commit: payload.commit, parentIndex: payload.parentIndex ?? 0, id: payload.id, side: payload.side ?? 'right', signal});
     case 'vcs/tree-file':

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@feng0531/dsh-vcs",
-  "version": "0.6.1",
+  "version": "0.7.0",
   "private": false,
   "publishConfig": { "access": "public", "registry": "https://registry.npmjs.org/" },
   "type": "module",
@@ -31,6 +31,7 @@
     "src/commit-signature.mjs",
     "src/operation-budget.mjs",
     "src/historical-access.mjs",
+    "src/content-search.mjs",
     "src/preview-capabilities.mjs",
     "dist",
     "locale",

package/src/content-search.mjs

@@ -0,0 +1,74 @@
+import {createHash,createHmac,randomBytes,timingSafeEqual} from 'node:crypto';
+import {parseTreeEntries} from './historical-access.mjs';
+import {currentOperationBudget} from './operation-budget.mjs';
+
+const MAX_BYTES = 16 * 1024 * 1024;
+const secret = randomBytes(32);
+const invalid = () => Error('Invalid content search cursor');
+const binding = (root,snapshot,query) => createHash('sha256').update(JSON.stringify([root,snapshot,query])).digest('hex');
+const sign = body => createHmac('sha256',secret).update(body).digest('base64url');
+function encode(state) { const body=Buffer.from(JSON.stringify(state)).toString('base64url');return body+'.'+sign(body); }
+function decode(cursor,root,snapshot,query) {
+  if(typeof cursor!=='string'||cursor.length>4096)throw invalid();
+  const [body,mac,...extra]=cursor.split('.');
+  if(!body||!mac||extra.length||!/^[A-Za-z0-9_-]+$/.test(body)||!/^[A-Za-z0-9_-]{43}$/.test(mac)||!timingSafeEqual(Buffer.from(mac),Buffer.from(sign(body))))throw invalid();
+  let state;try{state=JSON.parse(Buffer.from(body,'base64url').toString());}catch{throw invalid();}
+  if(state.binding!==binding(root,snapshot,query)||!Number.isSafeInteger(state.index)||state.index<0||!Number.isSafeInteger(state.offset)||state.offset<0||state.offset>MAX_BYTES||!Number.isSafeInteger(state.line)||state.line<1||typeof state.skipped!=='boolean')throw invalid();
+  return state;
+}
+
+/** Stable object/tree positions, not an offset into a capped grep result set. */
+export async function searchContentPage(run,root,{snapshot,query,cursor,limit=100,signal}) {
+  if(!Number.isInteger(limit)||limit<1||limit>100)throw Error('Invalid content search page size');
+  const check=()=>{signal?.throwIfAborted();currentOperationBudget()?.throwIfExpired();};
+  check();
+  const state=cursor===undefined?{binding:binding(root,snapshot,query),index:0,offset:0,line:1,skipped:false}:decode(cursor,root,snapshot,query);
+  const listing=await run(root,['ls-tree','-r','-z','--full-tree',snapshot],MAX_BYTES,{signal});
+  const entries=parseTreeEntries(listing).filter(entry=>entry.type==='blob'&&['100644','100755'].includes(entry.mode));
+  if(state.index>entries.length)throw invalid();
+  const matches=[],needle=Buffer.from(query),warnings=[];
+  let scannedBytes=0,responseBytes=0,files=0,reason=null;
+  while(state.index<entries.length) {
+    check();
+    if(matches.length>=limit){reason='page-limit';break;}
+    if(files>=64||scannedBytes>=MAX_BYTES){reason='scan-budget';break;}
+    const entry=entries[state.index];
+    const rawSize=(await run(root,['cat-file','-s',entry.oid],2097152,{signal})).toString().trim();
+    if(!/^\d+$/.test(rawSize)||!Number.isSafeInteger(Number(rawSize)))throw Error('Invalid search object size');
+    const size=Number(rawSize);
+    if(size>MAX_BYTES){state.skipped=true;state.index++;state.offset=0;state.line=1;files++;continue;}
+    if(files&&scannedBytes+size>MAX_BYTES){reason='scan-budget';break;}
+    const body=await run(root,['cat-file','blob',entry.oid],Math.max(1,size),{signal});
+    scannedBytes+=body.length;files++;
+    if(body.length!==size||state.offset>size)throw Error('Search object size mismatch');
+    // Match git grep -I: binary files are outside this text-search domain.
+    if(body.includes(0)){state.index++;state.offset=0;state.line=1;continue;}
+    let pos=state.offset,line=state.line;
+    while(pos<body.length) {
+      check();
+      // Single pass: CR-only input must not rescan the remaining blob for LF per line.
+      let end=pos;
+      while(end<body.length&&body[end]!==10&&body[end]!==13){
+        if((end & 65535)===0)check();
+        end++;
+      }
+      const hit=body.subarray(pos,end).indexOf(needle);
+      if(hit>=0) {
+        let textEnd=Math.min(end,pos+4096);
+        while(textEnd<end&&(body[textEnd]&192)===128)textEnd--;
+        const match={path:entry.path,line,text:body.subarray(pos,textEnd).toString('utf8'),...(textEnd<end?{textTruncated:true}:{})};
+        const matchBytes=Buffer.byteLength(JSON.stringify(match));
+        if(responseBytes+matchBytes>1024*1024){reason='page-limit';break;}
+        matches.push(match);responseBytes+=matchBytes;
+      }
+      pos=end<body.length?end+(body[end]===13&&body[end+1]===10?2:1):end;line++;
+      if(matches.length>=limit)break;
+    }
+    if(pos>=body.length){state.index++;state.offset=0;state.line=1;}
+    else {state.offset=pos;state.line=line;reason='page-limit';break;}
+  }
+  check();
+  const more=state.index<entries.length;
+  if(state.skipped)warnings.push('Files exceeding the 16 MiB text-search limit were skipped.');
+  return {snapshot,query,matches,nextCursor:more?encode(state):null,complete:!more&&!state.skipped,truncated:more||state.skipped,reason:reason??(state.skipped?'unsupported-content':null),warnings};
+}

package/src/historical-access.mjs

@@ -1,10 +1,33 @@
 import {isObjectId} from '../git-history.mjs';
+import {currentOperationBudget} from './operation-budget.mjs';
 
 export const HISTORICAL_CACHE_LIMITS = Object.freeze({maxBytes: 32 * 1024 * 1024, maxEntries: 8, ttlMs: 60000});
 const MAX_BLOB_BYTES = 16 * 1024 * 1024;
 const MAX_LIST_BYTES = 16 * 1024 * 1024;
 const MAX_DIRECTORY_ENTRIES = 100000;
 const CHUNK_BYTES = 64 * 1024;
+const isLineBreak = (buffer,pos) => buffer[pos] === 10 || (buffer[pos] === 13 && buffer[pos + 1] !== 10);
+// At most 257 checkpoints for a 16 MiB blob, independent of its line count.
+function buildLineIndex(buffer,signal) {
+  const index=[];let line=1,lineStart=0;
+  for(let offset=0;offset<=buffer.length;offset++) {
+    if(offset%CHUNK_BYTES===0){signal?.throwIfAborted();currentOperationBudget()?.throwIfExpired();index.push({offset,line,lineStart});}
+    if(isLineBreak(buffer,offset)){line++;lineStart=offset+1;}
+  }
+  return {index,totalLines:line};
+}
+function locateOffset(buffer,index,offset) {
+  const point=index[Math.floor(offset/CHUNK_BYTES)];let line=point.line,lineStart=point.lineStart;
+  for(let pos=point.offset;pos<offset;pos++)if(isLineBreak(buffer,pos)){line++;lineStart=pos+1;}
+  return {line,lineStart};
+}
+function locateLine(buffer,index,line) {
+  let point=index[0];for(const candidate of index){if(candidate.line>line)break;point=candidate;}
+  if(point.line===line)return point.lineStart;
+  let current=point.line;
+  for(let pos=point.offset;pos<buffer.length;pos++)if(isLineBreak(buffer,pos)&&++current===line)return pos+1;
+  return buffer.length;
+}
 
 export function validateHistoricalPath(file, allowRoot = false) {
   if (typeof file !== 'string' || file.length > 32768 || (!file && !allowRoot) || file.includes('\0') || file.startsWith('/') || file.split('/').some(part => (!part || part === '.' || part === '..') && !(allowRoot && file === ''))) {
@@ -87,14 +110,17 @@ export function createHistoricalAccess({run, now = Date.now, maxBytes = HISTORIC
     filtered.sort((a,b) => Number(b.type === 'tree') - Number(a.type === 'tree') || (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
     return {commit,path:directory,query,entries:filtered.slice(offset,offset + limit),nextOffset:offset + limit < filtered.length ? offset + limit : null};
   };
-  const segment = async (root, {commit,path:file,offset = 0,signal,expectedEpoch = epoch} = {}) => {
+  const segment = async (root, {commit,path:file,offset:requestedOffset,line:targetLine,signal,expectedEpoch = epoch} = {}) => {
     const generation = expectedEpoch; // Includes caller repository validation and all member lookups.
+    if(targetLine !== undefined && (!Number.isSafeInteger(targetLine) || targetLine < 1 || targetLine > MAX_BLOB_BYTES + 1 || requestedOffset !== undefined)) throw Error('Invalid segment target line');
+    let offset = requestedOffset ?? 0;
     if (!Number.isInteger(offset) || offset < 0 || offset > MAX_BLOB_BYTES) throw Error('Invalid segment offset');
     const entry = await lookup(root, {commit,path:file,signal,missingMessage:'Segment requires a selected committed regular file'});
     if (entry.type !== 'blob' || !['100644','100755'].includes(entry.mode)) throw Error('Segment requires a selected committed regular file');
     purge();
     const key = JSON.stringify([root,commit,entry.oid]);
-    let buffer = generation === epoch ? cache.get(key)?.buffer : undefined;
+    let cached = generation === epoch ? cache.get(key) : undefined;
+    let buffer = cached?.buffer, lineIndex = cached?.lineIndex;
     if (buffer) cacheHits++;
     if (!buffer) {
       const rawSize = (await run(root, ['cat-file','-s',entry.oid], 2097152, {signal})).toString('utf8').trim();
@@ -116,24 +142,32 @@ export function createHistoricalAccess({run, now = Date.now, maxBytes = HISTORIC
         if (buffer.includes(0) || buffer.some(byte => byte < 32 && ![9,10,12,13].includes(byte))) throw Error('Segments support UTF-8 text only');
         new TextDecoder('utf-8',{fatal:true}).decode(buffer);
         decodeCount++;
+        lineIndex = buildLineIndex(buffer,signal);
         if (generation === epoch) {
           remove(key);
           const timer = setTimeout(() => remove(key), ttlMs);
           timer.unref?.();
-          cache.set(key, {root,buffer,expiresAt:now() + ttlMs,timer});
+          cache.set(key, {root,buffer,lineIndex,expiresAt:now() + ttlMs,timer});
           bytes += buffer.length;
         }
       } finally {reservedBytes -= size; reservedEntries--;}
     }
     signal?.throwIfAborted();
     const size = buffer.length;
+    const targetFound = targetLine === undefined || targetLine <= lineIndex.totalLines;
+    if(targetLine !== undefined) offset = targetFound ? locateLine(buffer,lineIndex.index,targetLine) : size;
     if (offset > size) throw Error('Segment offset exceeds file size');
+    const start = locateOffset(buffer,lineIndex.index,offset);
     if (offset < size && (buffer[offset] & 192) === 128) throw Error('Segment offset splits UTF-8 character');
     let end = Math.min(size, offset + CHUNK_BYTES);
     while (end < size && (buffer[end] & 192) === 128) end--;
+    if(end < size && buffer[end - 1] === 13 && buffer[end] === 10) end--;
     const text = new TextDecoder('utf-8',{fatal:true,ignoreBOM:true}).decode(buffer.subarray(offset,end));
     signal?.throwIfAborted();
-    return {commit,path:file,oid:entry.oid,encoding:'UTF-8',offset,endOffset:end,totalBytes:size,nextOffset:end < size ? end : null,text};
+    const endLine = locateOffset(buffer,lineIndex.index,end).line;
+    const endsMidLine = end < size && end > offset && !isLineBreak(buffer,end - 1);
+    const lineTruncated = endsMidLine;
+    return {commit,path:file,oid:entry.oid,encoding:'UTF-8',offset,endOffset:end,totalBytes:size,nextOffset:end < size ? end : null,text,startLine:start.line,endLine,startsMidLine:offset > start.lineStart,targetFound,lineTruncated,endsMidLine,...(!targetFound?{notice:'Requested line exceeds file line count.'}:{})};
   };
   return {lookup,treePage,segment,clear,captureEpoch:() => epoch,stats:() => {purge();return {bytes,entries:cache.size,reservedBytes,reservedEntries,blobReads,blobReadBytes,decodeCount,cacheHits};}};
 }

package/vcs.mjs

@@ -15,6 +15,8 @@ import {inside, confined} from './repository-path.mjs';
 import {readWorkspaceLargeText} from './workspace-large-text.mjs';
 import {createHistoricalAccess,validateHistoricalPath} from './src/historical-access.mjs';
 import {withComparisonPreview} from './src/preview-capabilities.mjs';
+import {searchContentPage} from './src/content-search.mjs';
+import {withOperationBudget} from './src/operation-budget.mjs';
 
 const historicalAccess = createHistoricalAccess({run:git});
 /** Host invalidates object bytes on grant replacement, expiry and plugin disposal. */
@@ -272,35 +274,18 @@ export async function listStashes(repo,{signal}={}){
  return {stashes};
 }
 
-export async function searchRepositoryContent(repo,{commit,query,signal}={}) {
+export async function searchRepositoryContent(repo,{commit,query,cursor,limit=100,signal}={}) {
+ return withOperationBudget({signal},async()=>{
  signal?.throwIfAborted();
  if(typeof query!=='string'||!query||query.length>1024||/[\0\r\n]/.test(query))throw new Error('Invalid content search query');
+ if(cursor!==undefined&&!commit)throw Error('Content search continuation requires a fixed commit');
  repo=await checkedGitRepo(repo,signal,'Content search supports Git only');
  let snapshot=commit;
- if(!snapshot){
-  try{snapshot=(await git(repo.root,['rev-parse','--verify','HEAD^{commit}'],MAX_TEXT,{signal})).toString('utf8').trim();}
-  catch{throw new Error('Content search requires a resolved HEAD commit');}
- }
+ if(!snapshot) snapshot=await resolveHeadCommit(args=>git(repo.root,args,MAX_TEXT,{signal}));
  if(!isObjectId(snapshot))throw new Error('Invalid content search commit');
  if((await git(repo.root,['cat-file','-t',snapshot],MAX_TEXT,{signal})).toString('utf8').trim()!=='commit')throw Error('Content search requires a commit object');
- let text;
- try{text=(await git(repo.root,['-c','grep.column=false','grep','--no-color','-F','-n','-I','-z','--max-count=100','-e',query,snapshot,'--'],MAX_TEXT,{signal})).toString('utf8');}
- catch(error){if(error.code==='VCS_COMMAND'&&error.exitCode===1)return {snapshot,matches:[]};throw error;}
- signal?.throwIfAborted();
- const matches=[];
- // -z frames path and line with NUL; legal paths can contain LF.
- for(let position=0;position<text.length;){
-  const pathEnd=text.indexOf(String.fromCharCode(0),position),lineEnd=text.indexOf(String.fromCharCode(0),pathEnd+1),recordEnd=text.indexOf(String.fromCharCode(10),lineEnd+1);
-  if(pathEnd<position||lineEnd<0||recordEnd<0)throw new Error('Invalid content search result');
-  const treePath=text.slice(position,pathEnd),lineNo=text.slice(pathEnd+1,lineEnd),lineText=text.slice(lineEnd+1,recordEnd);position=recordEnd+1;
-  const cut=treePath.indexOf(':');
-  if(cut<0)throw new Error('Invalid content search result');
-  const path=treePath.slice(cut+1);
-  if(!path||!/^\d+$/.test(lineNo))throw new Error('Invalid content search result');
-  matches.push({path,line:Number(lineNo),text:lineText});
-  if(matches.length>500)throw new Error('Content search exceeds match limit');
- }
- return {snapshot,matches};
+ return searchContentPage(git,repo.root,{snapshot,query,cursor,limit,signal});
+ });
 }
 
 export async function getCommitDetails(repo,options={}) {
@@ -349,11 +334,13 @@ function parseCommitChanges(text,scope){
   }
   return changes;
 }
-export async function getHistoricalSegment(repo, {commit,path:file,offset = 0,signal} = {}) {
+export async function getHistoricalSegment(repo, {commit,path:file,offset,line,signal} = {}) {
+ return withOperationBudget({signal},async()=>{
  const expectedEpoch = historicalAccess.captureEpoch();
  // Capture before any await: revoked in-flight validation cannot refill object bytes.
  repo = await checkedGitRepo(repo, signal, 'Historical segments support Git only');
- return historicalAccess.segment(repo.root, {commit,path:file,offset,signal,expectedEpoch});
+ return historicalAccess.segment(repo.root, {commit,path:file,offset,line,signal,expectedEpoch});
+ });
 }
 
 export async function getHistoricalFile(repo,{commit,path:file,signal}={}) {