@allansantos-dev/smart-tool 0.9.0 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +50 -23
- package/NOTICE +32 -32
- package/README.md +117 -116
- package/atomic_io.py +94 -94
- package/bin/smart-tool.mjs +5 -1
- package/client_hooks.py +192 -192
- package/code_graph_java.mjs +49 -49
- package/code_graph_js.cjs +135 -135
- package/duplicates.py +313 -313
- package/embedding_cache.py +99 -99
- package/endpoint_sync.py +179 -179
- package/gateway.py +220 -220
- package/hook_decision.py +96 -96
- package/index_inventory.py +75 -75
- package/indexer.py +878 -878
- package/install.py +113 -11
- package/install_runtime.py +101 -35
- package/integration_metrics.py +82 -82
- package/local_embedder.py +172 -172
- package/model_defaults.py +70 -70
- package/package.json +35 -35
- package/project_insights_ui.py +38 -38
- package/project_monitor.py +204 -204
- package/project_store.py +220 -220
- package/projects_ui.py +146 -146
- package/requirements.txt +8 -8
- package/research_cache.py +171 -171
- package/research_metrics.py +110 -110
- package/router.py +582 -582
- package/secret_store.py +121 -121
- package/setup_ui.py +1011 -986
- package/smart_tool_daemon.py +3604 -3604
- package/version.py +1 -1
- package/web_adapters/browser/fetch_camoufox.py +69 -0
- package/web_adapters/node/ddg-search.mjs +89 -89
- package/web_adapters/node/open-websearch-adapter.mjs +77 -77
- package/web_adapters/node/package.json +12 -12
- package/web_adapters/node/wikipedia-search.mjs +44 -44
- package/web_search_adapters.py +193 -193
- package/web_search_external.py +288 -288
- package/web_search_health.py +88 -88
package/project_insights_ui.py
CHANGED
|
@@ -1,38 +1,38 @@
|
|
|
1
|
-
"""Painéis por projeto: consumo, armazenamento seletivo e comparação de visões."""
|
|
2
|
-
STYLE=r'''
|
|
3
|
-
.insights-controls{display:flex;gap:10px;align-items:end;flex-wrap:wrap;margin:16px 0}.insights-controls label{font-size:12px;margin:0 0 4px}.insights-controls select{max-width:100%;min-height:40px}.insights-table-wrap{max-height:340px;overflow:auto;margin:16px 0}.insights-table{width:100%;border-collapse:collapse;font-size:13px}.insights-table th,.insights-table td{text-align:left;vertical-align:top;padding:10px 9px;border-bottom:1px solid var(--border);overflow-wrap:anywhere}.insights-table th{color:var(--muted);font-weight:600}.insights-table td:first-child,.insights-table th:first-child{padding-left:0}.insights-table td button{font-size:12px;min-height:30px;padding:4px 8px}.insights-note{color:var(--muted);font-size:13px;overflow-wrap:anywhere}.insights-summary{display:grid;grid-template-columns:1fr 1fr;gap:0 20px;margin:14px 0}.insights-summary div{padding:10px 0;border-top:1px solid var(--border)}.insights-summary small{display:block}.insights-summary strong{font-size:15px}.cleanup-preview{border:1px solid var(--border);border-radius:8px;padding:16px;margin:16px 0}.comparison-diff{white-space:pre-wrap;overflow-wrap:anywhere;font:12px/1.65 Consolas,monospace;max-height:430px;overflow:auto}.comparison-diff .added{color:var(--good)}.comparison-diff .removed{color:var(--bad)}.comparison-item{width:100%;text-align:left;margin:4px 0}.insights-note:empty{display:none}.insights-panel select{min-width:0}.insights-views{display:grid;grid-template-columns:1fr 1fr;gap:16px}.insights-views select{width:100%}
|
|
4
|
-
@media(max-width:700px){.insights-summary,.insights-views{grid-template-columns:1fr}.insights-table{min-width:490px}.insights-controls>div{max-width:100%}}
|
|
5
|
-
'''
|
|
6
|
-
HTML=r'''
|
|
7
|
-
<details class="insights-panel" id="usage-panel"><summary>Usage and reuse by project and view</summary>
|
|
8
|
-
<div class="insights-controls"><div><label for="usage-days">Period</label><select id="usage-days"><option value="7">7 days</option><option value="30" selected>30 days</option><option value="90">90 days</option><option value="366">1 year</option></select></div><div><label for="usage-view">View</label><select id="usage-view"><option value="">All</option></select></div><button id="usage-load">Refresh usage</button></div><p id="usage-message" class="insights-note">Local counts and usage reported by the gateway. No query text is saved in this telemetry.</p><div id="usage-summary" class="insights-summary"></div><div id="usage-table" class="insights-table-wrap"></div><div id="usage-indexing" class="insights-table-wrap"></div></details>
|
|
9
|
-
<details class="insights-panel" id="storage-panel"><summary>Disk space and view cleanup</summary><div class="insights-controls"><button id="storage-load">Refresh inventory</button></div><p class="insights-note">The active view and pinned views are protected. Cleanup is manual and removes only Smart Tool indexes and cache.</p><div id="storage-summary" class="insights-summary"></div><div id="storage-table" class="insights-table-wrap"></div><label class="check-label"><input id="cleanup-cache" type="checkbox" checked>Also clear vectors no longer used by the stored generations</label><div class="actions"><button id="cleanup-preview" disabled>Preview selected cleanup</button></div><div id="cleanup-plan" class="cleanup-preview" hidden></div><button id="cleanup-apply" class="danger" hidden>Confirm removal of this preview</button><p id="storage-message" class="insights-note"></p></details>
|
|
10
|
-
<details class="insights-panel" id="compare-panel"><summary>Compare two stored views</summary><p class="insights-note">The comparison uses the index content, with no checkout and no new embeddings. Scope differences can also show up as added or removed files.</p><button id="compare-load-views">Load views</button><div class="insights-views"><div><label for="compare-left">View A · base</label><select id="compare-left"></select></div><div><label for="compare-right">View B · compare with A</label><select id="compare-right"></select></div></div><label class="check-label"><input id="compare-relations" type="checkbox" checked>Also compare resolved references and static calls</label><div class="actions"><button id="compare-run" disabled>Compare A → B</button></div><p id="compare-message" class="insights-note"></p><div id="compare-summary" class="insights-summary"></div><div id="compare-files" class="insights-table-wrap"></div><div id="compare-graph"></div><h3 id="compare-file-title"></h3><pre id="compare-diff" class="comparison-diff"></pre></details>
|
|
11
|
-
'''
|
|
12
|
-
SCRIPT=r'''
|
|
13
|
-
window.SmartInsights=(()=>{
|
|
14
|
-
const $i=id=>document.getElementById(id);let sequence=0,storage=null,plan=null,comparison=null,compareRequest=0,diffRequest=0;
|
|
15
|
-
const count=n=>Number(n||0).toLocaleString('en-US'),size=n=>n>=1048576?(n/1048576).toLocaleString('en-US',{maximumFractionDigits:2})+' MiB':(n/1024).toLocaleString('en-US',{maximumFractionDigits:1})+' KiB';
|
|
16
|
-
const phaseNames={index_embedding:'Indexing',query_embedding:'Query',embedding_probe:'Model check',scope:'Scope',rerank:'Rerank',other:'Other step'};
|
|
17
|
-
function summary(host,rows){host.innerHTML=rows.map(([title,value,note])=>`<div><small>${esc(title)}</small><strong>${esc(value)}</strong>${note?'<small>'+esc(note)+'</small>':''}</div>`).join('')}
|
|
18
|
-
function table(headers,rows){return '<table class="insights-table"><thead><tr>'+headers.map(x=>'<th>'+esc(x)+'</th>').join('')+'</tr></thead><tbody>'+rows.map(row=>'<tr>'+row.map(value=>'<td>'+value+'</td>').join('')+'</tr>').join('')+'</tbody></table>'}
|
|
19
|
-
async function run(button,task){const seq=sequence,key=selected;button.disabled=true;try{await task(key,()=>seq===sequence&&key===selected)}catch(e){if(seq===sequence)message(e.message,true)}finally{if(seq===sequence)button.disabled=false}}
|
|
20
|
-
function clearPlan(){plan=null;$i('cleanup-plan').hidden=true;$i('cleanup-apply').hidden=true;$i('cleanup-preview').disabled=!document.querySelector('#storage-table input[data-storage]:checked')}
|
|
21
|
-
function reset(){sequence++;storage=null;comparison=null;plan=null;for(const id of ['usage-summary','usage-table','usage-indexing','storage-summary','storage-table','storage-message','compare-summary','compare-files','compare-graph','compare-file-title','compare-diff','compare-message'])$i(id).replaceChildren();$i('usage-view').replaceChildren(new Option('All',''));$i('compare-left').replaceChildren();$i('compare-right').replaceChildren();$i('compare-run').disabled=true;clearPlan();for(const id of ['usage-load','storage-load','compare-load-views'])$i(id).disabled=false;$i('usage-message').textContent='Refresh the usage of this project. The telemetry stores only counts and metadata.'}
|
|
22
|
-
async function loadUsage(key,valid){const requestedDays=$i('usage-days').value,requestedView=$i('usage-view').value;const data=await api({action:'usage',project_id:key,days:Number($i('usage-days').value),...($i('usage-view').value?{view_id:$i('usage-view').value}:{})});if(!valid()||requestedDays!==$i('usage-days').value||requestedView!==$i('usage-view').value)return;const t=data.totals,old=$i('usage-view').value,views=new Map();for(const row of [...data.daily,...data.indexing])views.set(row.view_id,row.view_label);if(!old)$i('usage-view').replaceChildren(new Option('All',''),...Array.from(views,([id,label])=>new Option(label||id,id)));summary($i('usage-summary'),[['Call attempts',count(t.attempts),count(t.errors)+' with errors'],['Reported tokens',t.token_reports?count((t.prompt_tokens||0)+(t.output_tokens||0)):'Not reported',count(t.token_reports)+' of '+count(t.attempts)+' calls with data'],['Cost reported by the gateway',t.reported_usd==null?'Not reported':new Intl.NumberFormat('en-US',{style:'currency',currency:'USD',maximumFractionDigits:6}).format(t.reported_usd),count(t.price_reports)+' calls with a cost; not the full bill'],['Chunks sent / cache',count(data.indexing.reduce((s,r)=>s+r.embedded_chunks,0))+' / '+count(data.indexing.reduce((s,r)=>s+r.reused_chunks,0)),'Unchanged files are counted separately']]);$i('usage-message').textContent='Loaded period: '+data.days+' days (UTC). '+data.notes.join(' ');$i('usage-table').innerHTML=table(['Date UTC','View / step','Model','Attempts','Reported tokens','Measured characters'],data.daily.slice(0,200).map(r=>[esc(r.day),esc(r.view_label)+'<br>'+esc(phaseNames[r.purpose]||r.purpose),esc(r.model),count(r.attempts),r.token_reports?count(r.prompt_tokens+r.output_tokens):'Not reported',count(r.input_characters)]));$i('usage-indexing').innerHTML=table(['View','Indexing jobs','Chunks sent','Cache reuse','Unchanged files'],data.indexing.map(r=>[esc(r.view_label),count(r.jobs),count(r.embedded_chunks),count(r.reused_chunks),count(r.unchanged_files)]))}
|
|
23
|
-
function renderStorage(data){storage=data;clearPlan();summary($i('storage-summary'),[['SQLite indexes',size(data.total_bytes)],['Vector cache',size(data.cache_bytes)],['Auxiliary generations',size(data.auxiliary_bytes)],['Stored views',count(data.views.length),'Active and pinned views are protected']]);$i('storage-table').innerHTML=table(['Clear','View','Space','Protection'],data.views.map(v=>[`<input type="checkbox" data-storage="${esc(v.storage_id)}" aria-label="Select ${esc(v.label)} for cleanup" ${v.protected?'disabled':''}>`,esc(v.label)+'<br><small>'+esc(v.commit?.slice(0,10)||'Commit not recorded')+'</small>',size(v.bytes),esc(v.protected_reason||'Can be selected')+`<br><button data-pin="${esc(v.storage_id)}" data-pinned="${v.pinned?'1':'0'}">${v.pinned?'Unpin':'Pin'}</button>`]));$i('storage-message').textContent=data.views.length?'Choose the old views and check the preview before removing.':'No stored views in this project.';clearPlan()}
|
|
24
|
-
async function loadStorage(key,valid){const data=await api({action:'storage',project_id:key});if(valid())renderStorage(data)}
|
|
25
|
-
$i('usage-load').onclick=()=>run($i('usage-load'),loadUsage);$i('usage-days').onchange=()=>run($i('usage-load'),loadUsage);$i('usage-view').onchange=()=>run($i('usage-load'),loadUsage);$i('storage-load').onclick=()=>run($i('storage-load'),loadStorage);
|
|
26
|
-
$i('storage-table').onchange=clearPlan;$i('cleanup-cache').onchange=clearPlan;
|
|
27
|
-
$i('storage-table').onclick=e=>{const button=e.target.closest('[data-pin]');if(button)run(button,async(key,valid)=>{const data=await api({action:'pin_view',project_id:key,storage_id:button.dataset.pin,pinned:button.dataset.pinned!=='1'});if(valid())renderStorage(data)})};
|
|
28
|
-
$i('cleanup-preview').onclick=()=>run($i('cleanup-preview'),async(key,valid)=>{const ids=[...document.querySelectorAll('#storage-table input[data-storage]:checked')].map(n=>n.dataset.storage);const data=await api({action:'cleanup_preview',project_id:key,storage_ids:ids,clear_cache:$i('cleanup-cache').checked});if(!valid())return;plan=data;$i('cleanup-plan').hidden=false;$i('cleanup-plan').innerHTML='<h3>Removal preview</h3><ul>'+data.views.map(v=>'<li>'+esc(v.label)+'</li>').join('')+'</ul><p>'+size(data.index_bytes)+' in indexes; '+count(data.unused_vectors)+' unreferenced vectors ('+size(data.vector_bytes)+' of vector content).</p><p class="insights-note">'+esc(data.note||'The final cache size depends on SQLite compaction.')+'</p><p>The project files will be kept.</p>';$i('cleanup-apply').hidden=false});
|
|
29
|
-
$i('cleanup-apply').onclick=()=>{if(!plan)return;if(!confirm('Remove the views in this preview: '+plan.views.map(v=>v.label).join(', ')+'? The project files will be kept.'))return;run($i('cleanup-apply'),async(key,valid)=>{const data=await api({action:'cleanup_commit',project_id:key,plan_id:plan.plan_id,confirm:true});if(!valid())return;renderStorage(data.storage);$i('storage-message').textContent=data.removed_views+' view(s) removed; '+data.removed_vectors+' vectors removed. '+data.warnings.join(' ');window.SmartMaps?.reset();await refresh()})};
|
|
30
|
-
async function loadViews(key,valid){compareRequest++;diffRequest++;comparison=null;for(const id of ['compare-summary','compare-files','compare-graph','compare-file-title','compare-diff'])$i(id).replaceChildren();const data=await api({action:'storage',project_id:key});if(!valid())return;const options=data.views.map(v=>[v.storage_id,v.label+(v.current?' (active)':'')]);$i('compare-left').replaceChildren(...options.map(([id,label])=>new Option(label,id)));$i('compare-right').replaceChildren(...options.map(([id,label])=>new Option(label,id)));if(options.length>1)$i('compare-right').selectedIndex=1;$i('compare-run').disabled=options.length<2;$i('compare-message').textContent=options.length<2?'Two indexed views are needed. Run searches on the branches you want to store them.':'Differences will be shown from A to B.'}
|
|
31
|
-
$i('compare-load-views').onclick=()=>run($i('compare-load-views'),loadViews);
|
|
32
|
-
for(const id of ['compare-left','compare-right','compare-relations'])$i(id).onchange=()=>{compareRequest++;diffRequest++;comparison=null;for(const host of ['compare-summary','compare-files','compare-graph','compare-file-title','compare-diff'])$i(host).replaceChildren();$i('compare-run').disabled=!$i('compare-left').value||!$i('compare-right').value||$i('compare-left').value===$i('compare-right').value;$i('compare-message').textContent='Selection changed. Run the comparison from A to B.'};
|
|
33
|
-
function fileButtons(data){const rows=[];for(const kind of ['added','removed','changed'])for(const path of data.files[kind])rows.push([kind==='added'?'Added in B':kind==='removed'?'Missing in B':'Changed',path]);for(const item of data.files.renamed)rows.push(['Rename detected',item.from+' → '+item.to]);return table(['Change','File'],rows.slice(0,1000).map(([kind,path])=>[esc(kind),kind==='Rename detected'?esc(path):`<button data-compare-file="${esc(path)}">${esc(path)}</button>`]))}
|
|
34
|
-
$i('compare-run').onclick=()=>run($i('compare-run'),async(key,valid)=>{const request=++compareRequest;diffRequest++;const left=$i('compare-left').value,right=$i('compare-right').value;if(!left||!right||left===right)throw Error('Choose two different views.');$i('compare-message').textContent='Comparing the stored snapshots…';const data=await api({action:'compare',project_id:key,left_view:left,right_view:right,relations:$i('compare-relations').checked});if(!valid()||request!==compareRequest)return;comparison={left,right};summary($i('compare-summary'),[['Added in B',count(data.files.added.length)],['Missing in B',count(data.files.removed.length)],['Changed',count(data.files.changed.length)],['Renamed / identical',count(data.files.renamed.length)+' / '+count(data.unchanged)]]);$i('compare-files').innerHTML=fileButtons(data);$i('compare-message').textContent=data.notes.join(' ')+(data.analysis_partial?' Some files/relations could not be analyzed.':'');$i('compare-file-title').textContent='';$i('compare-diff').textContent='';$i('compare-graph').innerHTML=data.relations?Object.entries(data.relations).map(([kind,changes])=>'<details><summary>'+esc(kind==='calls'?'Calls':'References between files')+' · +'+changes.added_count+' / −'+changes.removed_count+'</summary><div class="insights-table-wrap">'+table(['Change','Source → target'],[...changes.added.map(x=>['Added',esc(x[0])+' → '+esc(x[1])]),...changes.removed.map(x=>['Removed',esc(x[0])+' → '+esc(x[1])])])+'</div></details>').join(''):''});
|
|
35
|
-
$i('compare-files').onclick=e=>{const button=e.target.closest('[data-compare-file]');if(!button||!comparison)return;const chosen={...comparison},request=++diffRequest;run(button,async(key,valid)=>{const data=await api({action:'compare',project_id:key,left_view:chosen.left,right_view:chosen.right,file_path:button.dataset.compareFile,relations:false});if(!valid()||request!==diffRequest||comparison?.left!==chosen.left||comparison?.right!==chosen.right)return;$i('compare-file-title').textContent=button.dataset.compareFile+(button.dataset.compareFile.toLowerCase().endsWith('.docx')?' · extracted text':'');$i('compare-diff').innerHTML=data.diff_error?esc(data.diff_error):(data.diff||'The indexed text is identical; metadata/hashes may have changed.').split('\n').map(line=>'<span class="'+(line.startsWith('+')?'added':line.startsWith('-')?'removed':'')+'">'+esc(line)+'</span>').join('\n');if(data.diff_truncated)$i('compare-message').textContent+=' Diff limited to 1,000 lines / 128,000 characters.'})};
|
|
36
|
-
return {reset};
|
|
37
|
-
})();
|
|
38
|
-
'''
|
|
1
|
+
"""Painéis por projeto: consumo, armazenamento seletivo e comparação de visões."""
|
|
2
|
+
STYLE=r'''
|
|
3
|
+
.insights-controls{display:flex;gap:10px;align-items:end;flex-wrap:wrap;margin:16px 0}.insights-controls label{font-size:12px;margin:0 0 4px}.insights-controls select{max-width:100%;min-height:40px}.insights-table-wrap{max-height:340px;overflow:auto;margin:16px 0}.insights-table{width:100%;border-collapse:collapse;font-size:13px}.insights-table th,.insights-table td{text-align:left;vertical-align:top;padding:10px 9px;border-bottom:1px solid var(--border);overflow-wrap:anywhere}.insights-table th{color:var(--muted);font-weight:600}.insights-table td:first-child,.insights-table th:first-child{padding-left:0}.insights-table td button{font-size:12px;min-height:30px;padding:4px 8px}.insights-note{color:var(--muted);font-size:13px;overflow-wrap:anywhere}.insights-summary{display:grid;grid-template-columns:1fr 1fr;gap:0 20px;margin:14px 0}.insights-summary div{padding:10px 0;border-top:1px solid var(--border)}.insights-summary small{display:block}.insights-summary strong{font-size:15px}.cleanup-preview{border:1px solid var(--border);border-radius:8px;padding:16px;margin:16px 0}.comparison-diff{white-space:pre-wrap;overflow-wrap:anywhere;font:12px/1.65 Consolas,monospace;max-height:430px;overflow:auto}.comparison-diff .added{color:var(--good)}.comparison-diff .removed{color:var(--bad)}.comparison-item{width:100%;text-align:left;margin:4px 0}.insights-note:empty{display:none}.insights-panel select{min-width:0}.insights-views{display:grid;grid-template-columns:1fr 1fr;gap:16px}.insights-views select{width:100%}
|
|
4
|
+
@media(max-width:700px){.insights-summary,.insights-views{grid-template-columns:1fr}.insights-table{min-width:490px}.insights-controls>div{max-width:100%}}
|
|
5
|
+
'''
|
|
6
|
+
HTML=r'''
|
|
7
|
+
<details class="insights-panel" id="usage-panel"><summary>Usage and reuse by project and view</summary>
|
|
8
|
+
<div class="insights-controls"><div><label for="usage-days">Period</label><select id="usage-days"><option value="7">7 days</option><option value="30" selected>30 days</option><option value="90">90 days</option><option value="366">1 year</option></select></div><div><label for="usage-view">View</label><select id="usage-view"><option value="">All</option></select></div><button id="usage-load">Refresh usage</button></div><p id="usage-message" class="insights-note">Local counts and usage reported by the gateway. No query text is saved in this telemetry.</p><div id="usage-summary" class="insights-summary"></div><div id="usage-table" class="insights-table-wrap"></div><div id="usage-indexing" class="insights-table-wrap"></div></details>
|
|
9
|
+
<details class="insights-panel" id="storage-panel"><summary>Disk space and view cleanup</summary><div class="insights-controls"><button id="storage-load">Refresh inventory</button></div><p class="insights-note">The active view and pinned views are protected. Cleanup is manual and removes only Smart Tool indexes and cache.</p><div id="storage-summary" class="insights-summary"></div><div id="storage-table" class="insights-table-wrap"></div><label class="check-label"><input id="cleanup-cache" type="checkbox" checked>Also clear vectors no longer used by the stored generations</label><div class="actions"><button id="cleanup-preview" disabled>Preview selected cleanup</button></div><div id="cleanup-plan" class="cleanup-preview" hidden></div><button id="cleanup-apply" class="danger" hidden>Confirm removal of this preview</button><p id="storage-message" class="insights-note"></p></details>
|
|
10
|
+
<details class="insights-panel" id="compare-panel"><summary>Compare two stored views</summary><p class="insights-note">The comparison uses the index content, with no checkout and no new embeddings. Scope differences can also show up as added or removed files.</p><button id="compare-load-views">Load views</button><div class="insights-views"><div><label for="compare-left">View A · base</label><select id="compare-left"></select></div><div><label for="compare-right">View B · compare with A</label><select id="compare-right"></select></div></div><label class="check-label"><input id="compare-relations" type="checkbox" checked>Also compare resolved references and static calls</label><div class="actions"><button id="compare-run" disabled>Compare A → B</button></div><p id="compare-message" class="insights-note"></p><div id="compare-summary" class="insights-summary"></div><div id="compare-files" class="insights-table-wrap"></div><div id="compare-graph"></div><h3 id="compare-file-title"></h3><pre id="compare-diff" class="comparison-diff"></pre></details>
|
|
11
|
+
'''
|
|
12
|
+
SCRIPT=r'''
|
|
13
|
+
window.SmartInsights=(()=>{
|
|
14
|
+
const $i=id=>document.getElementById(id);let sequence=0,storage=null,plan=null,comparison=null,compareRequest=0,diffRequest=0;
|
|
15
|
+
const count=n=>Number(n||0).toLocaleString('en-US'),size=n=>n>=1048576?(n/1048576).toLocaleString('en-US',{maximumFractionDigits:2})+' MiB':(n/1024).toLocaleString('en-US',{maximumFractionDigits:1})+' KiB';
|
|
16
|
+
const phaseNames={index_embedding:'Indexing',query_embedding:'Query',embedding_probe:'Model check',scope:'Scope',rerank:'Rerank',other:'Other step'};
|
|
17
|
+
function summary(host,rows){host.innerHTML=rows.map(([title,value,note])=>`<div><small>${esc(title)}</small><strong>${esc(value)}</strong>${note?'<small>'+esc(note)+'</small>':''}</div>`).join('')}
|
|
18
|
+
function table(headers,rows){return '<table class="insights-table"><thead><tr>'+headers.map(x=>'<th>'+esc(x)+'</th>').join('')+'</tr></thead><tbody>'+rows.map(row=>'<tr>'+row.map(value=>'<td>'+value+'</td>').join('')+'</tr>').join('')+'</tbody></table>'}
|
|
19
|
+
async function run(button,task){const seq=sequence,key=selected;button.disabled=true;try{await task(key,()=>seq===sequence&&key===selected)}catch(e){if(seq===sequence)message(e.message,true)}finally{if(seq===sequence)button.disabled=false}}
|
|
20
|
+
function clearPlan(){plan=null;$i('cleanup-plan').hidden=true;$i('cleanup-apply').hidden=true;$i('cleanup-preview').disabled=!document.querySelector('#storage-table input[data-storage]:checked')}
|
|
21
|
+
function reset(){sequence++;storage=null;comparison=null;plan=null;for(const id of ['usage-summary','usage-table','usage-indexing','storage-summary','storage-table','storage-message','compare-summary','compare-files','compare-graph','compare-file-title','compare-diff','compare-message'])$i(id).replaceChildren();$i('usage-view').replaceChildren(new Option('All',''));$i('compare-left').replaceChildren();$i('compare-right').replaceChildren();$i('compare-run').disabled=true;clearPlan();for(const id of ['usage-load','storage-load','compare-load-views'])$i(id).disabled=false;$i('usage-message').textContent='Refresh the usage of this project. The telemetry stores only counts and metadata.'}
|
|
22
|
+
async function loadUsage(key,valid){const requestedDays=$i('usage-days').value,requestedView=$i('usage-view').value;const data=await api({action:'usage',project_id:key,days:Number($i('usage-days').value),...($i('usage-view').value?{view_id:$i('usage-view').value}:{})});if(!valid()||requestedDays!==$i('usage-days').value||requestedView!==$i('usage-view').value)return;const t=data.totals,old=$i('usage-view').value,views=new Map();for(const row of [...data.daily,...data.indexing])views.set(row.view_id,row.view_label);if(!old)$i('usage-view').replaceChildren(new Option('All',''),...Array.from(views,([id,label])=>new Option(label||id,id)));summary($i('usage-summary'),[['Call attempts',count(t.attempts),count(t.errors)+' with errors'],['Reported tokens',t.token_reports?count((t.prompt_tokens||0)+(t.output_tokens||0)):'Not reported',count(t.token_reports)+' of '+count(t.attempts)+' calls with data'],['Cost reported by the gateway',t.reported_usd==null?'Not reported':new Intl.NumberFormat('en-US',{style:'currency',currency:'USD',maximumFractionDigits:6}).format(t.reported_usd),count(t.price_reports)+' calls with a cost; not the full bill'],['Chunks sent / cache',count(data.indexing.reduce((s,r)=>s+r.embedded_chunks,0))+' / '+count(data.indexing.reduce((s,r)=>s+r.reused_chunks,0)),'Unchanged files are counted separately']]);$i('usage-message').textContent='Loaded period: '+data.days+' days (UTC). '+data.notes.join(' ');$i('usage-table').innerHTML=table(['Date UTC','View / step','Model','Attempts','Reported tokens','Measured characters'],data.daily.slice(0,200).map(r=>[esc(r.day),esc(r.view_label)+'<br>'+esc(phaseNames[r.purpose]||r.purpose),esc(r.model),count(r.attempts),r.token_reports?count(r.prompt_tokens+r.output_tokens):'Not reported',count(r.input_characters)]));$i('usage-indexing').innerHTML=table(['View','Indexing jobs','Chunks sent','Cache reuse','Unchanged files'],data.indexing.map(r=>[esc(r.view_label),count(r.jobs),count(r.embedded_chunks),count(r.reused_chunks),count(r.unchanged_files)]))}
|
|
23
|
+
function renderStorage(data){storage=data;clearPlan();summary($i('storage-summary'),[['SQLite indexes',size(data.total_bytes)],['Vector cache',size(data.cache_bytes)],['Auxiliary generations',size(data.auxiliary_bytes)],['Stored views',count(data.views.length),'Active and pinned views are protected']]);$i('storage-table').innerHTML=table(['Clear','View','Space','Protection'],data.views.map(v=>[`<input type="checkbox" data-storage="${esc(v.storage_id)}" aria-label="Select ${esc(v.label)} for cleanup" ${v.protected?'disabled':''}>`,esc(v.label)+'<br><small>'+esc(v.commit?.slice(0,10)||'Commit not recorded')+'</small>',size(v.bytes),esc(v.protected_reason||'Can be selected')+`<br><button data-pin="${esc(v.storage_id)}" data-pinned="${v.pinned?'1':'0'}">${v.pinned?'Unpin':'Pin'}</button>`]));$i('storage-message').textContent=data.views.length?'Choose the old views and check the preview before removing.':'No stored views in this project.';clearPlan()}
|
|
24
|
+
async function loadStorage(key,valid){const data=await api({action:'storage',project_id:key});if(valid())renderStorage(data)}
|
|
25
|
+
$i('usage-load').onclick=()=>run($i('usage-load'),loadUsage);$i('usage-days').onchange=()=>run($i('usage-load'),loadUsage);$i('usage-view').onchange=()=>run($i('usage-load'),loadUsage);$i('storage-load').onclick=()=>run($i('storage-load'),loadStorage);
|
|
26
|
+
$i('storage-table').onchange=clearPlan;$i('cleanup-cache').onchange=clearPlan;
|
|
27
|
+
$i('storage-table').onclick=e=>{const button=e.target.closest('[data-pin]');if(button)run(button,async(key,valid)=>{const data=await api({action:'pin_view',project_id:key,storage_id:button.dataset.pin,pinned:button.dataset.pinned!=='1'});if(valid())renderStorage(data)})};
|
|
28
|
+
$i('cleanup-preview').onclick=()=>run($i('cleanup-preview'),async(key,valid)=>{const ids=[...document.querySelectorAll('#storage-table input[data-storage]:checked')].map(n=>n.dataset.storage);const data=await api({action:'cleanup_preview',project_id:key,storage_ids:ids,clear_cache:$i('cleanup-cache').checked});if(!valid())return;plan=data;$i('cleanup-plan').hidden=false;$i('cleanup-plan').innerHTML='<h3>Removal preview</h3><ul>'+data.views.map(v=>'<li>'+esc(v.label)+'</li>').join('')+'</ul><p>'+size(data.index_bytes)+' in indexes; '+count(data.unused_vectors)+' unreferenced vectors ('+size(data.vector_bytes)+' of vector content).</p><p class="insights-note">'+esc(data.note||'The final cache size depends on SQLite compaction.')+'</p><p>The project files will be kept.</p>';$i('cleanup-apply').hidden=false});
|
|
29
|
+
$i('cleanup-apply').onclick=()=>{if(!plan)return;if(!confirm('Remove the views in this preview: '+plan.views.map(v=>v.label).join(', ')+'? The project files will be kept.'))return;run($i('cleanup-apply'),async(key,valid)=>{const data=await api({action:'cleanup_commit',project_id:key,plan_id:plan.plan_id,confirm:true});if(!valid())return;renderStorage(data.storage);$i('storage-message').textContent=data.removed_views+' view(s) removed; '+data.removed_vectors+' vectors removed. '+data.warnings.join(' ');window.SmartMaps?.reset();await refresh()})};
|
|
30
|
+
async function loadViews(key,valid){compareRequest++;diffRequest++;comparison=null;for(const id of ['compare-summary','compare-files','compare-graph','compare-file-title','compare-diff'])$i(id).replaceChildren();const data=await api({action:'storage',project_id:key});if(!valid())return;const options=data.views.map(v=>[v.storage_id,v.label+(v.current?' (active)':'')]);$i('compare-left').replaceChildren(...options.map(([id,label])=>new Option(label,id)));$i('compare-right').replaceChildren(...options.map(([id,label])=>new Option(label,id)));if(options.length>1)$i('compare-right').selectedIndex=1;$i('compare-run').disabled=options.length<2;$i('compare-message').textContent=options.length<2?'Two indexed views are needed. Run searches on the branches you want to store them.':'Differences will be shown from A to B.'}
|
|
31
|
+
$i('compare-load-views').onclick=()=>run($i('compare-load-views'),loadViews);
|
|
32
|
+
for(const id of ['compare-left','compare-right','compare-relations'])$i(id).onchange=()=>{compareRequest++;diffRequest++;comparison=null;for(const host of ['compare-summary','compare-files','compare-graph','compare-file-title','compare-diff'])$i(host).replaceChildren();$i('compare-run').disabled=!$i('compare-left').value||!$i('compare-right').value||$i('compare-left').value===$i('compare-right').value;$i('compare-message').textContent='Selection changed. Run the comparison from A to B.'};
|
|
33
|
+
function fileButtons(data){const rows=[];for(const kind of ['added','removed','changed'])for(const path of data.files[kind])rows.push([kind==='added'?'Added in B':kind==='removed'?'Missing in B':'Changed',path]);for(const item of data.files.renamed)rows.push(['Rename detected',item.from+' → '+item.to]);return table(['Change','File'],rows.slice(0,1000).map(([kind,path])=>[esc(kind),kind==='Rename detected'?esc(path):`<button data-compare-file="${esc(path)}">${esc(path)}</button>`]))}
|
|
34
|
+
$i('compare-run').onclick=()=>run($i('compare-run'),async(key,valid)=>{const request=++compareRequest;diffRequest++;const left=$i('compare-left').value,right=$i('compare-right').value;if(!left||!right||left===right)throw Error('Choose two different views.');$i('compare-message').textContent='Comparing the stored snapshots…';const data=await api({action:'compare',project_id:key,left_view:left,right_view:right,relations:$i('compare-relations').checked});if(!valid()||request!==compareRequest)return;comparison={left,right};summary($i('compare-summary'),[['Added in B',count(data.files.added.length)],['Missing in B',count(data.files.removed.length)],['Changed',count(data.files.changed.length)],['Renamed / identical',count(data.files.renamed.length)+' / '+count(data.unchanged)]]);$i('compare-files').innerHTML=fileButtons(data);$i('compare-message').textContent=data.notes.join(' ')+(data.analysis_partial?' Some files/relations could not be analyzed.':'');$i('compare-file-title').textContent='';$i('compare-diff').textContent='';$i('compare-graph').innerHTML=data.relations?Object.entries(data.relations).map(([kind,changes])=>'<details><summary>'+esc(kind==='calls'?'Calls':'References between files')+' · +'+changes.added_count+' / −'+changes.removed_count+'</summary><div class="insights-table-wrap">'+table(['Change','Source → target'],[...changes.added.map(x=>['Added',esc(x[0])+' → '+esc(x[1])]),...changes.removed.map(x=>['Removed',esc(x[0])+' → '+esc(x[1])])])+'</div></details>').join(''):''});
|
|
35
|
+
$i('compare-files').onclick=e=>{const button=e.target.closest('[data-compare-file]');if(!button||!comparison)return;const chosen={...comparison},request=++diffRequest;run(button,async(key,valid)=>{const data=await api({action:'compare',project_id:key,left_view:chosen.left,right_view:chosen.right,file_path:button.dataset.compareFile,relations:false});if(!valid()||request!==diffRequest||comparison?.left!==chosen.left||comparison?.right!==chosen.right)return;$i('compare-file-title').textContent=button.dataset.compareFile+(button.dataset.compareFile.toLowerCase().endsWith('.docx')?' · extracted text':'');$i('compare-diff').innerHTML=data.diff_error?esc(data.diff_error):(data.diff||'The indexed text is identical; metadata/hashes may have changed.').split('\n').map(line=>'<span class="'+(line.startsWith('+')?'added':line.startsWith('-')?'removed':'')+'">'+esc(line)+'</span>').join('\n');if(data.diff_truncated)$i('compare-message').textContent+=' Diff limited to 1,000 lines / 128,000 characters.'})};
|
|
36
|
+
return {reset};
|
|
37
|
+
})();
|
|
38
|
+
'''
|
package/project_monitor.py
CHANGED
|
@@ -1,204 +1,204 @@
|
|
|
1
|
-
"""Eventos Windows + reconciliação: fila apenas para projetos habilitados pelo usuário."""
|
|
2
|
-
import os
|
|
3
|
-
import threading
|
|
4
|
-
import time
|
|
5
|
-
|
|
6
|
-
import config
|
|
7
|
-
import directory_watch
|
|
8
|
-
import indexer
|
|
9
|
-
import local_embedder
|
|
10
|
-
import index_scope
|
|
11
|
-
import project_store
|
|
12
|
-
import index_views
|
|
13
|
-
|
|
14
|
-
DEBOUNCE_S = 5
|
|
15
|
-
RECONCILE_S = 180
|
|
16
|
-
FULL_CHECK_S = 86400
|
|
17
|
-
MAX_EVENT_PATHS = 1000
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
def relevant_path(path):
|
|
21
|
-
parts = path.replace("\\", "/").strip("/").split("/")
|
|
22
|
-
return bool(parts and all(part not in ("", ".", "..") and not part.startswith(".")
|
|
23
|
-
and not index_scope.always_excluded(part) for part in parts))
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
def snapshot(root, scope):
|
|
27
|
-
result = {}
|
|
28
|
-
for rel in index_scope.resolve_included_files(root, scope):
|
|
29
|
-
stat, _reason = indexer._stat_candidate(root, rel)
|
|
30
|
-
if stat is not None:
|
|
31
|
-
result[rel] = stat
|
|
32
|
-
return result
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
def index_model(current=None):
|
|
37
|
-
return local_embedder.resolve(config.load_config().get("embedding_model"), current)
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
def needs_model_rebuild(meta, configured_model):
|
|
41
|
-
return meta.get("model_id") != configured_model or not meta.get("exists")
|
|
42
|
-
|
|
43
|
-
class ProjectMonitor:
|
|
44
|
-
def __init__(self, enqueue, watcher_factory=directory_watch.DirectoryWatcher):
|
|
45
|
-
self.enqueue = enqueue
|
|
46
|
-
self.watcher_factory = watcher_factory
|
|
47
|
-
self._watchers = {}
|
|
48
|
-
self._retry_watch = {}
|
|
49
|
-
self._checked = {}
|
|
50
|
-
self._git_seen = {}
|
|
51
|
-
self._git_checked = {}
|
|
52
|
-
self._pending = {}
|
|
53
|
-
self._lock = threading.RLock()
|
|
54
|
-
self._stop = threading.Event()
|
|
55
|
-
self._thread = None
|
|
56
|
-
|
|
57
|
-
def start(self):
|
|
58
|
-
self._thread = threading.Thread(target=self._run, name="project-monitor", daemon=True)
|
|
59
|
-
self._thread.start()
|
|
60
|
-
|
|
61
|
-
def stop(self):
|
|
62
|
-
self._stop.set()
|
|
63
|
-
if self._thread:
|
|
64
|
-
self._thread.join(timeout=3)
|
|
65
|
-
with self._lock:
|
|
66
|
-
watchers = list(self._watchers.values())
|
|
67
|
-
self._watchers.clear()
|
|
68
|
-
for watcher in watchers:
|
|
69
|
-
watcher.stop()
|
|
70
|
-
|
|
71
|
-
def refresh(self):
|
|
72
|
-
self._checked.clear()
|
|
73
|
-
|
|
74
|
-
def events(self, project_id, rows, overflow=False):
|
|
75
|
-
project = project_store.get(project_id)
|
|
76
|
-
if not project or project.get("paused") or not project.get("watch") or not project.get("enabled"):
|
|
77
|
-
return
|
|
78
|
-
paths = {row["path"] for row in rows if relevant_path(row.get("path", ""))}
|
|
79
|
-
if not paths and not overflow:
|
|
80
|
-
return
|
|
81
|
-
# Um diretório existente também recebe MODIFIED ao mudar um arquivo filho.
|
|
82
|
-
# Só uma estrutura nova ou documentação da raiz exige reavaliar o escopo.
|
|
83
|
-
scope = index_scope.load_scope(project["root"]) or {}
|
|
84
|
-
known_dirs = {p.casefold() for p in scope.get("structure", [])}
|
|
85
|
-
docs = {name.casefold() for name in index_scope._ROOT_DOC_NAMES}
|
|
86
|
-
force_scope = any(path.casefold() in docs for path in paths) or any(
|
|
87
|
-
row.get("action") in (1, 5) and row.get("path") in paths
|
|
88
|
-
and row["path"].casefold() not in known_dirs
|
|
89
|
-
and os.path.isdir(os.path.join(project["root"], row["path"])) for row in rows)
|
|
90
|
-
force_scope = force_scope and not scope.get("manual", False)
|
|
91
|
-
with self._lock:
|
|
92
|
-
first = project_id not in self._pending
|
|
93
|
-
state = self._pending.setdefault(project_id, {"paths": set(), "full": False, "force_scope": False, "revision": 0})
|
|
94
|
-
state["revision"] = state.get("revision", 0) + 1
|
|
95
|
-
state["paths"].update(paths)
|
|
96
|
-
state["last"] = time.monotonic()
|
|
97
|
-
state["full"] |= overflow or len(state["paths"]) > MAX_EVENT_PATHS
|
|
98
|
-
state["force_scope"] |= force_scope
|
|
99
|
-
if state["full"]:
|
|
100
|
-
state["paths"].clear()
|
|
101
|
-
project_store.mark_dirty(project_id, "Changes detected by the monitor.")
|
|
102
|
-
project_store.record_changes(project_id, paths, full=overflow, scope=force_scope)
|
|
103
|
-
|
|
104
|
-
def _watch_error(self, project_id, message):
|
|
105
|
-
project_store.update(project_id, watch_backend="reconciliation", watch_error=message[:300])
|
|
106
|
-
self._retry_watch[project_id] = time.monotonic() + RECONCILE_S
|
|
107
|
-
self.events(project_id, [], overflow=True)
|
|
108
|
-
|
|
109
|
-
def _sync_watchers(self, projects, now):
|
|
110
|
-
wanted = {p["id"]: p for p in projects if p.get("watch") and p.get("enabled")
|
|
111
|
-
and not p.get("paused") and os.path.isdir(p["root"])}
|
|
112
|
-
for key in list(self._watchers):
|
|
113
|
-
if key not in wanted:
|
|
114
|
-
watcher = self._watchers.pop(key)
|
|
115
|
-
watcher.stop()
|
|
116
|
-
project_store.update(key, watch_backend="off")
|
|
117
|
-
for key, project in wanted.items():
|
|
118
|
-
current = self._watchers.get(key)
|
|
119
|
-
if current and current.alive:
|
|
120
|
-
continue
|
|
121
|
-
if now < self._retry_watch.get(key, 0):
|
|
122
|
-
continue
|
|
123
|
-
watcher = self.watcher_factory(
|
|
124
|
-
project["root"], lambda rows, overflow, k=key: self.events(k, rows, overflow),
|
|
125
|
-
lambda error, k=key: self._watch_error(k, error),
|
|
126
|
-
)
|
|
127
|
-
self._watchers[key] = watcher
|
|
128
|
-
project_store.update(key, watch_backend="windows_events", watch_error="")
|
|
129
|
-
watcher.start()
|
|
130
|
-
|
|
131
|
-
def tick(self):
|
|
132
|
-
now = time.monotonic()
|
|
133
|
-
projects = project_store.all_projects()
|
|
134
|
-
self._sync_watchers(projects, now)
|
|
135
|
-
for project in projects:
|
|
136
|
-
key = project["id"]
|
|
137
|
-
if not project.get("watch") or not project.get("enabled") or project.get("paused"):
|
|
138
|
-
continue
|
|
139
|
-
if not os.path.isdir(project["root"]):
|
|
140
|
-
project_store.update(key, status="unavailable", last_error="The folder is not accessible.")
|
|
141
|
-
continue
|
|
142
|
-
if any(job.get("status") == "pending" for job in project_store.jobs(key, limit=50)):
|
|
143
|
-
continue # Não planejar outra geração com metadados de uma ainda em curso.
|
|
144
|
-
if now - self._git_checked.get(key, -5) >= 5:
|
|
145
|
-
view = index_views.describe(project['root'], fresh=True)
|
|
146
|
-
signature = (view['view_id'], view.get('commit'))
|
|
147
|
-
if key in self._git_seen and signature != self._git_seen[key]:
|
|
148
|
-
self.events(key, [], overflow=True)
|
|
149
|
-
self._git_seen[key] = signature
|
|
150
|
-
self._git_checked[key] = now
|
|
151
|
-
if now - self._checked.get(key, -RECONCILE_S) >= RECONCILE_S:
|
|
152
|
-
self._checked[key] = now
|
|
153
|
-
scope = index_scope.load_scope(project["root"])
|
|
154
|
-
meta = indexer.info(project["root"])
|
|
155
|
-
rescope = scope is None or index_scope.needs_rescan(project["root"], scope)
|
|
156
|
-
changed = rescope
|
|
157
|
-
if scope is not None:
|
|
158
|
-
changed |= snapshot(project["root"], scope) != indexer.manifest_snapshot(project["root"])
|
|
159
|
-
full = time.time() - meta.get("full_scan_at", 0) >= FULL_CHECK_S
|
|
160
|
-
changed |= needs_model_rebuild(meta, index_model(meta.get("model_id")))
|
|
161
|
-
project_store.update(key, last_checked=time.time())
|
|
162
|
-
if changed or full or project.get("dirty_seq", 0) > project.get("indexed_seq", 0):
|
|
163
|
-
with self._lock:
|
|
164
|
-
first = key not in self._pending
|
|
165
|
-
state = self._pending.setdefault(key, {"paths": set(), "last": now - DEBOUNCE_S,
|
|
166
|
-
"full": False, "force_scope": False, "revision": 0})
|
|
167
|
-
state["full"] |= full
|
|
168
|
-
state["force_scope"] |= rescope
|
|
169
|
-
if first:
|
|
170
|
-
project_store.mark_dirty(key, "Reconciliation found a pending update.")
|
|
171
|
-
project_store.record_changes(key, full=full, scope=rescope)
|
|
172
|
-
# Monitoramento local é independente do gasto com o modelo. O padrão
|
|
173
|
-
# somente registra pendência; a próxima busca executa o diff e embeddings.
|
|
174
|
-
if project.get('update_mode', 'on_search') != 'eager':
|
|
175
|
-
with self._lock:
|
|
176
|
-
self._pending.pop(key, None)
|
|
177
|
-
continue
|
|
178
|
-
with self._lock:
|
|
179
|
-
pending = self._pending.get(key)
|
|
180
|
-
if not pending or now - pending["last"] < DEBOUNCE_S:
|
|
181
|
-
continue
|
|
182
|
-
options = {"changed_paths": sorted(pending["paths"]), "full_check": pending["full"],
|
|
183
|
-
"force_scope": pending["force_scope"]}
|
|
184
|
-
revision = pending.get("revision", 0)
|
|
185
|
-
current = project_store.get(key)
|
|
186
|
-
if time.time() < current.get("next_retry_at", 0):
|
|
187
|
-
continue
|
|
188
|
-
try:
|
|
189
|
-
self.enqueue(key, **options)
|
|
190
|
-
except Exception as exc:
|
|
191
|
-
project_store.update(key, status="error", last_error=str(exc)[:300], next_retry_at=time.time() + 60)
|
|
192
|
-
continue
|
|
193
|
-
with self._lock:
|
|
194
|
-
# Um evento recebido durante o submit continua pendente para a próxima geração.
|
|
195
|
-
if self._pending.get(key) is pending and pending.get("revision", 0) == revision:
|
|
196
|
-
self._pending.pop(key, None)
|
|
197
|
-
|
|
198
|
-
def _run(self):
|
|
199
|
-
while not self._stop.is_set():
|
|
200
|
-
try:
|
|
201
|
-
self.tick()
|
|
202
|
-
except Exception:
|
|
203
|
-
pass # Falha opcional do monitor não derruba o daemon MCP.
|
|
204
|
-
self._stop.wait(1)
|
|
1
|
+
"""Eventos Windows + reconciliação: fila apenas para projetos habilitados pelo usuário."""
|
|
2
|
+
import os
|
|
3
|
+
import threading
|
|
4
|
+
import time
|
|
5
|
+
|
|
6
|
+
import config
|
|
7
|
+
import directory_watch
|
|
8
|
+
import indexer
|
|
9
|
+
import local_embedder
|
|
10
|
+
import index_scope
|
|
11
|
+
import project_store
|
|
12
|
+
import index_views
|
|
13
|
+
|
|
14
|
+
DEBOUNCE_S = 5
|
|
15
|
+
RECONCILE_S = 180
|
|
16
|
+
FULL_CHECK_S = 86400
|
|
17
|
+
MAX_EVENT_PATHS = 1000
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def relevant_path(path):
|
|
21
|
+
parts = path.replace("\\", "/").strip("/").split("/")
|
|
22
|
+
return bool(parts and all(part not in ("", ".", "..") and not part.startswith(".")
|
|
23
|
+
and not index_scope.always_excluded(part) for part in parts))
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def snapshot(root, scope):
|
|
27
|
+
result = {}
|
|
28
|
+
for rel in index_scope.resolve_included_files(root, scope):
|
|
29
|
+
stat, _reason = indexer._stat_candidate(root, rel)
|
|
30
|
+
if stat is not None:
|
|
31
|
+
result[rel] = stat
|
|
32
|
+
return result
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def index_model(current=None):
|
|
37
|
+
return local_embedder.resolve(config.load_config().get("embedding_model"), current)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def needs_model_rebuild(meta, configured_model):
|
|
41
|
+
return meta.get("model_id") != configured_model or not meta.get("exists")
|
|
42
|
+
|
|
43
|
+
class ProjectMonitor:
|
|
44
|
+
def __init__(self, enqueue, watcher_factory=directory_watch.DirectoryWatcher):
|
|
45
|
+
self.enqueue = enqueue
|
|
46
|
+
self.watcher_factory = watcher_factory
|
|
47
|
+
self._watchers = {}
|
|
48
|
+
self._retry_watch = {}
|
|
49
|
+
self._checked = {}
|
|
50
|
+
self._git_seen = {}
|
|
51
|
+
self._git_checked = {}
|
|
52
|
+
self._pending = {}
|
|
53
|
+
self._lock = threading.RLock()
|
|
54
|
+
self._stop = threading.Event()
|
|
55
|
+
self._thread = None
|
|
56
|
+
|
|
57
|
+
def start(self):
|
|
58
|
+
self._thread = threading.Thread(target=self._run, name="project-monitor", daemon=True)
|
|
59
|
+
self._thread.start()
|
|
60
|
+
|
|
61
|
+
def stop(self):
|
|
62
|
+
self._stop.set()
|
|
63
|
+
if self._thread:
|
|
64
|
+
self._thread.join(timeout=3)
|
|
65
|
+
with self._lock:
|
|
66
|
+
watchers = list(self._watchers.values())
|
|
67
|
+
self._watchers.clear()
|
|
68
|
+
for watcher in watchers:
|
|
69
|
+
watcher.stop()
|
|
70
|
+
|
|
71
|
+
def refresh(self):
|
|
72
|
+
self._checked.clear()
|
|
73
|
+
|
|
74
|
+
def events(self, project_id, rows, overflow=False):
|
|
75
|
+
project = project_store.get(project_id)
|
|
76
|
+
if not project or project.get("paused") or not project.get("watch") or not project.get("enabled"):
|
|
77
|
+
return
|
|
78
|
+
paths = {row["path"] for row in rows if relevant_path(row.get("path", ""))}
|
|
79
|
+
if not paths and not overflow:
|
|
80
|
+
return
|
|
81
|
+
# Um diretório existente também recebe MODIFIED ao mudar um arquivo filho.
|
|
82
|
+
# Só uma estrutura nova ou documentação da raiz exige reavaliar o escopo.
|
|
83
|
+
scope = index_scope.load_scope(project["root"]) or {}
|
|
84
|
+
known_dirs = {p.casefold() for p in scope.get("structure", [])}
|
|
85
|
+
docs = {name.casefold() for name in index_scope._ROOT_DOC_NAMES}
|
|
86
|
+
force_scope = any(path.casefold() in docs for path in paths) or any(
|
|
87
|
+
row.get("action") in (1, 5) and row.get("path") in paths
|
|
88
|
+
and row["path"].casefold() not in known_dirs
|
|
89
|
+
and os.path.isdir(os.path.join(project["root"], row["path"])) for row in rows)
|
|
90
|
+
force_scope = force_scope and not scope.get("manual", False)
|
|
91
|
+
with self._lock:
|
|
92
|
+
first = project_id not in self._pending
|
|
93
|
+
state = self._pending.setdefault(project_id, {"paths": set(), "full": False, "force_scope": False, "revision": 0})
|
|
94
|
+
state["revision"] = state.get("revision", 0) + 1
|
|
95
|
+
state["paths"].update(paths)
|
|
96
|
+
state["last"] = time.monotonic()
|
|
97
|
+
state["full"] |= overflow or len(state["paths"]) > MAX_EVENT_PATHS
|
|
98
|
+
state["force_scope"] |= force_scope
|
|
99
|
+
if state["full"]:
|
|
100
|
+
state["paths"].clear()
|
|
101
|
+
project_store.mark_dirty(project_id, "Changes detected by the monitor.")
|
|
102
|
+
project_store.record_changes(project_id, paths, full=overflow, scope=force_scope)
|
|
103
|
+
|
|
104
|
+
def _watch_error(self, project_id, message):
|
|
105
|
+
project_store.update(project_id, watch_backend="reconciliation", watch_error=message[:300])
|
|
106
|
+
self._retry_watch[project_id] = time.monotonic() + RECONCILE_S
|
|
107
|
+
self.events(project_id, [], overflow=True)
|
|
108
|
+
|
|
109
|
+
def _sync_watchers(self, projects, now):
|
|
110
|
+
wanted = {p["id"]: p for p in projects if p.get("watch") and p.get("enabled")
|
|
111
|
+
and not p.get("paused") and os.path.isdir(p["root"])}
|
|
112
|
+
for key in list(self._watchers):
|
|
113
|
+
if key not in wanted:
|
|
114
|
+
watcher = self._watchers.pop(key)
|
|
115
|
+
watcher.stop()
|
|
116
|
+
project_store.update(key, watch_backend="off")
|
|
117
|
+
for key, project in wanted.items():
|
|
118
|
+
current = self._watchers.get(key)
|
|
119
|
+
if current and current.alive:
|
|
120
|
+
continue
|
|
121
|
+
if now < self._retry_watch.get(key, 0):
|
|
122
|
+
continue
|
|
123
|
+
watcher = self.watcher_factory(
|
|
124
|
+
project["root"], lambda rows, overflow, k=key: self.events(k, rows, overflow),
|
|
125
|
+
lambda error, k=key: self._watch_error(k, error),
|
|
126
|
+
)
|
|
127
|
+
self._watchers[key] = watcher
|
|
128
|
+
project_store.update(key, watch_backend="windows_events", watch_error="")
|
|
129
|
+
watcher.start()
|
|
130
|
+
|
|
131
|
+
def tick(self):
|
|
132
|
+
now = time.monotonic()
|
|
133
|
+
projects = project_store.all_projects()
|
|
134
|
+
self._sync_watchers(projects, now)
|
|
135
|
+
for project in projects:
|
|
136
|
+
key = project["id"]
|
|
137
|
+
if not project.get("watch") or not project.get("enabled") or project.get("paused"):
|
|
138
|
+
continue
|
|
139
|
+
if not os.path.isdir(project["root"]):
|
|
140
|
+
project_store.update(key, status="unavailable", last_error="The folder is not accessible.")
|
|
141
|
+
continue
|
|
142
|
+
if any(job.get("status") == "pending" for job in project_store.jobs(key, limit=50)):
|
|
143
|
+
continue # Não planejar outra geração com metadados de uma ainda em curso.
|
|
144
|
+
if now - self._git_checked.get(key, -5) >= 5:
|
|
145
|
+
view = index_views.describe(project['root'], fresh=True)
|
|
146
|
+
signature = (view['view_id'], view.get('commit'))
|
|
147
|
+
if key in self._git_seen and signature != self._git_seen[key]:
|
|
148
|
+
self.events(key, [], overflow=True)
|
|
149
|
+
self._git_seen[key] = signature
|
|
150
|
+
self._git_checked[key] = now
|
|
151
|
+
if now - self._checked.get(key, -RECONCILE_S) >= RECONCILE_S:
|
|
152
|
+
self._checked[key] = now
|
|
153
|
+
scope = index_scope.load_scope(project["root"])
|
|
154
|
+
meta = indexer.info(project["root"])
|
|
155
|
+
rescope = scope is None or index_scope.needs_rescan(project["root"], scope)
|
|
156
|
+
changed = rescope
|
|
157
|
+
if scope is not None:
|
|
158
|
+
changed |= snapshot(project["root"], scope) != indexer.manifest_snapshot(project["root"])
|
|
159
|
+
full = time.time() - meta.get("full_scan_at", 0) >= FULL_CHECK_S
|
|
160
|
+
changed |= needs_model_rebuild(meta, index_model(meta.get("model_id")))
|
|
161
|
+
project_store.update(key, last_checked=time.time())
|
|
162
|
+
if changed or full or project.get("dirty_seq", 0) > project.get("indexed_seq", 0):
|
|
163
|
+
with self._lock:
|
|
164
|
+
first = key not in self._pending
|
|
165
|
+
state = self._pending.setdefault(key, {"paths": set(), "last": now - DEBOUNCE_S,
|
|
166
|
+
"full": False, "force_scope": False, "revision": 0})
|
|
167
|
+
state["full"] |= full
|
|
168
|
+
state["force_scope"] |= rescope
|
|
169
|
+
if first:
|
|
170
|
+
project_store.mark_dirty(key, "Reconciliation found a pending update.")
|
|
171
|
+
project_store.record_changes(key, full=full, scope=rescope)
|
|
172
|
+
# Monitoramento local é independente do gasto com o modelo. O padrão
|
|
173
|
+
# somente registra pendência; a próxima busca executa o diff e embeddings.
|
|
174
|
+
if project.get('update_mode', 'on_search') != 'eager':
|
|
175
|
+
with self._lock:
|
|
176
|
+
self._pending.pop(key, None)
|
|
177
|
+
continue
|
|
178
|
+
with self._lock:
|
|
179
|
+
pending = self._pending.get(key)
|
|
180
|
+
if not pending or now - pending["last"] < DEBOUNCE_S:
|
|
181
|
+
continue
|
|
182
|
+
options = {"changed_paths": sorted(pending["paths"]), "full_check": pending["full"],
|
|
183
|
+
"force_scope": pending["force_scope"]}
|
|
184
|
+
revision = pending.get("revision", 0)
|
|
185
|
+
current = project_store.get(key)
|
|
186
|
+
if time.time() < current.get("next_retry_at", 0):
|
|
187
|
+
continue
|
|
188
|
+
try:
|
|
189
|
+
self.enqueue(key, **options)
|
|
190
|
+
except Exception as exc:
|
|
191
|
+
project_store.update(key, status="error", last_error=str(exc)[:300], next_retry_at=time.time() + 60)
|
|
192
|
+
continue
|
|
193
|
+
with self._lock:
|
|
194
|
+
# Um evento recebido durante o submit continua pendente para a próxima geração.
|
|
195
|
+
if self._pending.get(key) is pending and pending.get("revision", 0) == revision:
|
|
196
|
+
self._pending.pop(key, None)
|
|
197
|
+
|
|
198
|
+
def _run(self):
|
|
199
|
+
while not self._stop.is_set():
|
|
200
|
+
try:
|
|
201
|
+
self.tick()
|
|
202
|
+
except Exception:
|
|
203
|
+
pass # Falha opcional do monitor não derruba o daemon MCP.
|
|
204
|
+
self._stop.wait(1)
|