@crewx/search 0.1.10-rc.167 → 0.1.10-rc.169

package/dist/src/engine.js

@@ -1,3 +1,3 @@
-"use strict";var S=exports&&exports.__createBinding||(Object.create?(function(e,o,t,s){if(s===void 0)s=t;var n=Object.getOwnPropertyDescriptor(o,t);if(!n||("get"in n?!o.__esModule:n.writable||n.configurable)){n={enumerable:true,get:function(){return o[t]}}}Object.defineProperty(e,s,n)}):(function(e,o,t,s){if(s===void 0)s=t;e[s]=o[t]}));var P=exports&&exports.__setModuleDefault||(Object.create?(function(e,o){Object.defineProperty(e,"default",{enumerable:true,value:o})}):function(e,o){e["default"]=o});var _=exports&&exports.__importStar||(function(){var e=function(o){e=Object.getOwnPropertyNames||function(t){var s=[];for(var n in t)if(Object.prototype.hasOwnProperty.call(t,n))s[s.length]=n;return s};return e(o)};return function(o){if(o&&o.__esModule)return o;var t={};if(o!=null){for(var s=e(o),n=0;n<s.length;n++)if(s[n]!=="default")S(t,o,s[n])}P(t,o);return t}})();Object.defineProperty(exports,"__esModule",{value:true});exports.SearchEngine=void 0;const w=_(require("fs"));const m=_(require("path"));const B=require("glob");const p=require("@crewx/knowledge-core");const C=10*1024*1024;const h={okfTitle:5,highlightQuotes:4,okfDescription:3,highlightNotes:3,okfTags:2,highlightIntents:2,highlightColors:1.5,highlightedHeadings:1.5,okfLinks:1,body:1};function b(e){if(e==null)return"";if(typeof e==="string")return e;if(typeof e==="number"||typeof e==="boolean"||typeof e==="bigint"){return String(e)}if(e instanceof Date)return e.toISOString();if(Array.isArray(e)){return e.map(o=>b(o)).filter(Boolean).join(" ")}if(typeof e==="object"){return Object.values(e).map(o=>b(o)).filter(Boolean).join(" ")}return""}function N(e){const o=/\[[^\]]*\]\(([^)]+)\)/g;const t=[];let s;while((s=o.exec(e))!==null){if(s[1]&&!s[1].startsWith("http"))t.push(s[1])}return t.join(" ")}function O(e,o){const t={};if(e["type"])t.okfType=String(e["type"]);if(e["title"])t.okfTitle=b(e["title"]);if(e["description"])t.okfDescription=b(e["description"]);if(e["resource"])t.okfResource=String(e["resource"]);if(e["tags"])t.okfTags=b(e["tags"]);const s=N(o);if(s)t.okfLinks=s;const n=e["crewx"];if(n&&typeof n==="object"&&!Array.isArray(n)){const a=n["highlights"];if(Array.isArray(a)){const r=a.filter(l=>l!==null&&typeof l==="object"&&l.status!=="orphaned"&&l.status!=="deleted"&&l.status!=="superseded");const g=r.map(l=>l.quote??"").filter(Boolean).join(" ");const c=r.map(l=>l.note??"").filter(Boolean).join(" ");const i=r.map(l=>l.intent??"").filter(Boolean).join(" ");const u=r.map(l=>l.color??"").filter(Boolean).join(" ");const f=r.map(l=>l.context?.heading??"").filter(Boolean).join(" ");if(g)t.highlightQuotes=g;if(c)t.highlightNotes=c;if(i)t.highlightIntents=i;if(u)t.highlightColors=u;if(f)t.highlightedHeadings=f}}return t}function I(e,o){const t=o;if(e==="highlights"){return{okfTitle:h["okfTitle"],highlightQuotes:h["highlightQuotes"]*2*t,okfDescription:h["okfDescription"],highlightNotes:h["highlightNotes"]*2*t,okfTags:h["okfTags"],highlightIntents:h["highlightIntents"]*2*t,highlightColors:h["highlightColors"]*t,highlightedHeadings:h["highlightedHeadings"]*t,okfLinks:h["okfLinks"],body:h["body"]}}if(e==="body"){return{okfTitle:h["okfTitle"],highlightQuotes:1,okfDescription:h["okfDescription"],highlightNotes:1,okfTags:h["okfTags"],highlightIntents:.5,highlightColors:.5,highlightedHeadings:.5,okfLinks:h["okfLinks"],body:2}}return{okfTitle:h["okfTitle"],highlightQuotes:h["highlightQuotes"]*t,okfDescription:h["okfDescription"],highlightNotes:h["highlightNotes"]*t,okfTags:h["okfTags"],highlightIntents:h["highlightIntents"]*t,highlightColors:h["highlightColors"]*t,highlightedHeadings:h["highlightedHeadings"]*t,okfLinks:h["okfLinks"],body:h["body"]}}function L(e,o,t){const s=o.toLowerCase().split(/\s+/).filter(Boolean);const n=Object.entries(t).sort(([,r],[,g])=>g-r).map(([r])=>r);const a=e;for(const r of n){const g=String(a[r]??"").toLowerCase();if(g&&s.some(c=>g.includes(c))){return r}}return"body"}class A{async collectDocuments(o,t){const s=m.join(o,t);const n=(0,p.loadDocsConfig)((0,p.resolveProjectRoot)(o));const a=(0,p.buildIgnorePatterns)(n.exclude);const r=await(0,B.glob)(s,{nodir:true,ignore:a});const g=[];for(const c of r){try{const i=w.readFileSync(c,"utf-8");const u=m.relative(o,c);let f="";let l=i;let d={};if(c.endsWith(".md")){try{const k=(0,p.parseFrontmatter)(i);f=b(k.data["title"])||b(k.data["summary"]);l=k.content;d=O(k.data,l)}catch{const k=i.split("\n");f=(k[0]??"").replace(/^#\s*/,"").substring(0,100)}}else{f=m.basename(c)}g.push({file:u,fullPath:c,title:f,body:l.substring(0,C),okfTitle:d.okfTitle??f,...d})}catch{}}return{docs:g,fileCount:r.length}}runBm25Search(o,t,s,n="all",a=1){if(o.length<3)return[];const r=I(n,a);const g=o.map((i,u)=>({id:String(u),type:"document",category:"general",tags:[],date:"",summary:i.title,body:i.body,okfTitle:i.okfTitle??i.title,okfDescription:i.okfDescription??"",okfTags:i.okfTags??"",okfLinks:i.okfLinks??"",highlightQuotes:i.highlightQuotes??"",highlightNotes:i.highlightNotes??"",highlightIntents:i.highlightIntents??"",highlightColors:i.highlightColors??"",highlightedHeadings:i.highlightedHeadings??"",title:i.title,file:i.file,fullPath:i.fullPath}));const c=(0,p.searchEntries)(g,t,r);return c.slice(0,s).map(i=>{const u=o[Number(i.id)];const f={file:i["file"],fullPath:i["fullPath"],title:i["title"],body:i.body||"",score:i._score,okfTitle:i["okfTitle"],okfDescription:i["okfDescription"],okfTags:i["okfTags"],okfLinks:i["okfLinks"],highlightQuotes:i["highlightQuotes"],highlightNotes:i["highlightNotes"],highlightIntents:i["highlightIntents"],highlightColors:i["highlightColors"],highlightedHeadings:i["highlightedHeadings"],matchedField:u?L(u,t,r):"body"};return f})}async search(o){const{query:t,searchPath:s=".",glob:n="**/*.md",limit:a=20,json:r=false,scope:g="all",highlightBoost:c=1}=o;const i=m.resolve(s);const{docs:u,fileCount:f}=await this.collectDocuments(i,n);if(f===0){console.log(`No files found: ${m.join(i,n)}`);process.exit(1)}if(u.length===0){console.log("No readable files found.");process.exit(1)}const l=this.runBm25Search(u,t,a,g,c);if(l.length===0){console.log(`No results found for '${t}'.`);process.exit(0)}if(r){console.log(JSON.stringify(l.map(d=>({file:d.file,score:d.score,title:d.title.substring(0,100),matchedField:d.matchedField})),null,2))}else{console.log(`
+"use strict";var S=exports&&exports.__createBinding||(Object.create?(function(e,o,t,s){if(s===void 0)s=t;var n=Object.getOwnPropertyDescriptor(o,t);if(!n||("get"in n?!o.__esModule:n.writable||n.configurable)){n={enumerable:true,get:function(){return o[t]}}}Object.defineProperty(e,s,n)}):(function(e,o,t,s){if(s===void 0)s=t;e[s]=o[t]}));var P=exports&&exports.__setModuleDefault||(Object.create?(function(e,o){Object.defineProperty(e,"default",{enumerable:true,value:o})}):function(e,o){e["default"]=o});var _=exports&&exports.__importStar||(function(){var e=function(o){e=Object.getOwnPropertyNames||function(t){var s=[];for(var n in t)if(Object.prototype.hasOwnProperty.call(t,n))s[s.length]=n;return s};return e(o)};return function(o){if(o&&o.__esModule)return o;var t={};if(o!=null){for(var s=e(o),n=0;n<s.length;n++)if(s[n]!=="default")S(t,o,s[n])}P(t,o);return t}})();Object.defineProperty(exports,"__esModule",{value:true});exports.SearchEngine=void 0;const w=_(require("fs"));const m=_(require("path"));const B=require("glob");const p=require("@crewx/knowledge-core");const C=10*1024*1024;const h={okfTitle:5,highlightQuotes:4,okfDescription:3,highlightNotes:3,okfTags:2,highlightIntents:2,highlightColors:1.5,highlightedHeadings:1.5,okfLinks:1,body:1};function b(e){if(e==null)return"";if(typeof e==="string")return e;if(typeof e==="number"||typeof e==="boolean"||typeof e==="bigint"){return String(e)}if(e instanceof Date)return e.toISOString();if(Array.isArray(e)){return e.map(o=>b(o)).filter(Boolean).join(" ")}if(typeof e==="object"){return Object.values(e).map(o=>b(o)).filter(Boolean).join(" ")}return""}function N(e){const o=/\[[^\]]*\]\(([^)]+)\)/g;const t=[];let s;while((s=o.exec(e))!==null){if(s[1]&&!s[1].startsWith("http"))t.push(s[1])}return t.join(" ")}function O(e,o){const t={};if(e["type"])t.okfType=String(e["type"]);if(e["title"])t.okfTitle=b(e["title"]);if(e["description"])t.okfDescription=b(e["description"]);if(e["resource"])t.okfResource=String(e["resource"]);if(e["tags"])t.okfTags=b(e["tags"]);const s=N(o);if(s)t.okfLinks=s;const n=e["crewx"];if(n&&typeof n==="object"&&!Array.isArray(n)){const a=n["highlights"];if(Array.isArray(a)){const r=a.filter(l=>l!==null&&typeof l==="object"&&l.status!=="orphaned"&&l.status!=="deleted"&&l.status!=="superseded");const g=r.map(l=>l.quote??"").filter(Boolean).join(" ");const c=r.map(l=>l.note??"").filter(Boolean).join(" ");const i=r.map(l=>l.intent??"").filter(Boolean).join(" ");const u=r.map(l=>l.color??"").filter(Boolean).join(" ");const f=r.map(l=>l.context?.heading??"").filter(Boolean).join(" ");if(g)t.highlightQuotes=g;if(c)t.highlightNotes=c;if(i)t.highlightIntents=i;if(u)t.highlightColors=u;if(f)t.highlightedHeadings=f}}return t}function I(e,o){const t=o;if(e==="highlights"){return{okfTitle:h["okfTitle"],highlightQuotes:h["highlightQuotes"]*2*t,okfDescription:h["okfDescription"],highlightNotes:h["highlightNotes"]*2*t,okfTags:h["okfTags"],highlightIntents:h["highlightIntents"]*2*t,highlightColors:h["highlightColors"]*t,highlightedHeadings:h["highlightedHeadings"]*t,okfLinks:h["okfLinks"],body:h["body"]}}if(e==="body"){return{okfTitle:h["okfTitle"],highlightQuotes:1,okfDescription:h["okfDescription"],highlightNotes:1,okfTags:h["okfTags"],highlightIntents:.5,highlightColors:.5,highlightedHeadings:.5,okfLinks:h["okfLinks"],body:2}}return{okfTitle:h["okfTitle"],highlightQuotes:h["highlightQuotes"]*t,okfDescription:h["okfDescription"],highlightNotes:h["highlightNotes"]*t,okfTags:h["okfTags"],highlightIntents:h["highlightIntents"]*t,highlightColors:h["highlightColors"]*t,highlightedHeadings:h["highlightedHeadings"]*t,okfLinks:h["okfLinks"],body:h["body"]}}function L(e,o,t){const s=o.toLowerCase().split(/\s+/).filter(Boolean);const n=Object.entries(t).sort(([,r],[,g])=>g-r).map(([r])=>r);const a=e;for(const r of n){const g=String(a[r]??"").toLowerCase();if(g&&s.some(c=>g.includes(c))){return r}}return"body"}class A{async collectDocuments(o,t){const s=m.join(o,t);const n=(0,p.loadDocsConfig)((0,p.resolveProjectRoot)(o));const a=(0,p.buildIgnorePatterns)(n.exclude);const r=await(0,B.glob)(s,{nodir:true,ignore:a});const g=[];for(const c of r){try{const i=w.readFileSync(c,"utf-8");const u=m.relative(o,c);let f="";let l=i;let d={};if(c.endsWith(".md")){try{const k=(0,p.parseFrontmatter)(i);f=b(k.data["title"])||b(k.data["summary"]);l=k.content;d=O(k.data,l)}catch{const k=i.split("\n");f=(k[0]??"").replace(/^#\s*/,"").substring(0,100)}}else{f=m.basename(c)}g.push({file:u,fullPath:c,title:f,body:l.substring(0,C),okfTitle:d.okfTitle??f,...d})}catch{}}return{docs:g,fileCount:r.length}}runBm25Search(o,t,s,n="all",a=1){const r=I(n,a);const g=o.map((i,u)=>({id:String(u),type:"document",category:"general",tags:[],date:"",summary:i.title,body:i.body,okfTitle:i.okfTitle??i.title,okfDescription:i.okfDescription??"",okfTags:i.okfTags??"",okfLinks:i.okfLinks??"",highlightQuotes:i.highlightQuotes??"",highlightNotes:i.highlightNotes??"",highlightIntents:i.highlightIntents??"",highlightColors:i.highlightColors??"",highlightedHeadings:i.highlightedHeadings??"",title:i.title,file:i.file,fullPath:i.fullPath}));const c=(0,p.searchEntries)(g,t,r);return c.slice(0,s).map(i=>{const u=o[Number(i.id)];const f={file:i["file"],fullPath:i["fullPath"],title:i["title"],body:i.body||"",score:i._score,okfTitle:i["okfTitle"],okfDescription:i["okfDescription"],okfTags:i["okfTags"],okfLinks:i["okfLinks"],highlightQuotes:i["highlightQuotes"],highlightNotes:i["highlightNotes"],highlightIntents:i["highlightIntents"],highlightColors:i["highlightColors"],highlightedHeadings:i["highlightedHeadings"],matchedField:u?L(u,t,r):"body"};return f})}async search(o){const{query:t,searchPath:s=".",glob:n="**/*.md",limit:a=20,json:r=false,scope:g="all",highlightBoost:c=1}=o;const i=m.resolve(s);const{docs:u,fileCount:f}=await this.collectDocuments(i,n);if(f===0){console.log(`No files found: ${m.join(i,n)}`);process.exit(1)}if(u.length===0){console.log("No readable files found.");process.exit(1)}const l=this.runBm25Search(u,t,a,g,c);if(l.length===0){console.log(`No results found for '${t}'.`);process.exit(0)}if(r){console.log(JSON.stringify(l.map(d=>({file:d.file,score:d.score,title:d.title.substring(0,100),matchedField:d.matchedField})),null,2))}else{console.log(`
 \u{1F50D} Search: "${t}" (${l.length} results across ${f} files)
 `);for(const y of l){const j=y.title.substring(0,60).replace(/\n/g," ");const D=y.matchedField?` [${y.matchedField}]`:"";console.log(`[${y.score} pts] ${y.file}${D}`);if(j)console.log(`       ${j}`)}const d={commandId:"search",params:{query:t,path:i},resultCount:l.length};const k=(0,p.generateNextActions)(d);const T=(0,p.formatActions)(k);if(T)console.log(T)}}async searchAndReturn(o){const{query:t,searchPath:s=".",glob:n="**/*.md",limit:a=20,scope:r="all",highlightBoost:g=1}=o;const c=m.resolve(s);const{docs:i}=await this.collectDocuments(c,n);if(i.length===0){return[]}return this.runBm25Search(i,t,a,r,g)}}exports.SearchEngine=A;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crewx/search",
-  "version": "0.1.10-rc.167",
+  "version": "0.1.10-rc.169",
   "description": "BM25 full-text search engine for CrewX documents and code",
   "type": "commonjs",
   "main": "dist/src/engine.js",
@@ -34,8 +34,8 @@
   "license": "MIT",
   "dependencies": {
     "glob": "^10.0.0",
-    "@crewx/shared": "0.0.6-rc.86",
-    "@crewx/knowledge-core": "0.1.17-rc.181"
+    "@crewx/shared": "0.0.6-rc.88",
+    "@crewx/knowledge-core": "0.1.17-rc.183"
   },
   "devDependencies": {
     "@types/node": "^20.0.0",