clementine-agent 1.0.69 → 1.0.70

package/dist/brain/adapters/pdf.js

@@ -13,18 +13,24 @@ export async function* parsePdf(filePath) {
     try {
         buf = readFileSync(filePath);
     }
-    catch {
-        return;
+    catch (err) {
+        throw new Error(`Failed to read PDF ${path.basename(filePath)}: ${err instanceof Error ? err.message : String(err)}`);
     }
     let result;
     try {
         result = await pdfParse(buf);
     }
-    catch {
-        return;
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        const hint = /password/i.test(msg) ? ' (looks password-protected)' : '';
+        throw new Error(`Failed to parse PDF ${path.basename(filePath)}${hint}: ${msg}`);
     }
     const hint = path.basename(filePath, path.extname(filePath));
     const pages = splitPages(result.text);
+    const hasAnyText = pages.some((p) => p.trim().length > 0);
+    if (!hasAnyText) {
+        throw new Error(`PDF ${path.basename(filePath)} has no extractable text — likely image-only (OCR is not supported). Re-export with a text layer or transcribe it first.`);
+    }
     for (let i = 0; i < pages.length; i++) {
         const pageText = pages[i].trim();
         if (!pageText)

package/dist/brain/ingestion-pipeline.js

@@ -75,32 +75,41 @@ export async function runIngestion(opts) {
         const samples = [];
         const pendingStructured = [];
         for (const iter of recordIterators) {
-            for await (const record of iter) {
-                recordsIn += 1;
-                if (opts.limit && recordsIn > opts.limit)
-                    break;
-                const flowPath = classifyRecord(record, intelligenceMode);
-                if (flowPath === 'structured') {
-                    if (!schemaMapping && samples.length < SAMPLE_SIZE) {
-                        samples.push(record);
-                        pendingStructured.push(record);
-                        continue;
-                    }
-                    if (!schemaMapping && samples.length >= SAMPLE_SIZE) {
-                        schemaMapping = await inferSchema(samples, source.slug);
-                        await applyStructuredColumns(schemaMapping);
-                        for (const s of pendingStructured) {
-                            await processStructured(s, schemaMapping, source, opts, store, report, plannedRecords, errors, writtenSummaries, counters());
+            // Wrap so one unreadable file doesn't abort an otherwise-good folder
+            // ingest — record the adapter error and move on to the next iterator.
+            try {
+                for await (const record of iter) {
+                    recordsIn += 1;
+                    if (opts.limit && recordsIn > opts.limit)
+                        break;
+                    const flowPath = classifyRecord(record, intelligenceMode);
+                    if (flowPath === 'structured') {
+                        if (!schemaMapping && samples.length < SAMPLE_SIZE) {
+                            samples.push(record);
+                            pendingStructured.push(record);
+                            continue;
+                        }
+                        if (!schemaMapping && samples.length >= SAMPLE_SIZE) {
+                            schemaMapping = await inferSchema(samples, source.slug);
+                            await applyStructuredColumns(schemaMapping);
+                            for (const s of pendingStructured) {
+                                await processStructured(s, schemaMapping, source, opts, store, report, plannedRecords, errors, writtenSummaries, counters());
+                            }
+                            pendingStructured.length = 0;
+                        }
+                        if (schemaMapping) {
+                            await processStructured(record, schemaMapping, source, opts, store, report, plannedRecords, errors, writtenSummaries, counters());
                         }
-                        pendingStructured.length = 0;
                     }
-                    if (schemaMapping) {
-                        await processStructured(record, schemaMapping, source, opts, store, report, plannedRecords, errors, writtenSummaries, counters());
+                    else {
+                        await processFreeForm(record, source, opts, store, report, plannedRecords, errors, writtenSummaries, counters());
                     }
                 }
-                else {
-                    await processFreeForm(record, source, opts, store, report, plannedRecords, errors, writtenSummaries, counters());
-                }
+            }
+            catch (err) {
+                const msg = err instanceof Error ? err.message : String(err);
+                errors.push({ error: msg });
+                report('parsing', msg);
             }
         }
         // Flush structured records that never reached the schema-infer threshold
@@ -207,11 +216,22 @@ async function processStructured(record, mapping, source, opts, store, _report,
 }
 async function processFreeForm(record, source, opts, store, report, planned, _errors, _writtenSummaries, counters) {
     try {
-        report('distilling');
         const chunks = chunkContent(record.content, 3000);
-        const distillations = [];
-        for (const chunk of chunks) {
-            distillations.push(await distillChunk(chunk, record.metadata ?? {}));
+        report('distilling', chunks.length > 1 ? `chunk 0/${chunks.length}` : undefined);
+        // Parallelize per-chunk Haiku calls in small batches — one chunk at a
+        // time on a 30KB PDF adds up to 60–90s; 5-way concurrency cuts it to
+        // ~15s without pushing the API's rate limits.
+        const CONCURRENCY = 5;
+        const distillations = new Array(chunks.length);
+        let completed = 0;
+        for (let i = 0; i < chunks.length; i += CONCURRENCY) {
+            const batch = chunks.slice(i, i + CONCURRENCY);
+            const results = await Promise.all(batch.map((chunk) => distillChunk(chunk, record.metadata ?? {})));
+            for (let j = 0; j < results.length; j++)
+                distillations[i + j] = results[j];
+            completed += results.length;
+            if (chunks.length > 1)
+                report('distilling', `chunk ${completed}/${chunks.length}`);
         }
         const targetFolder = sanitizeFolder(source.targetFolder || `04-Ingest/${source.slug}`, source.slug);
         const partial = combineDistillations(record, distillations, source.slug, targetFolder);

package/dist/cli/dashboard.js

@@ -2483,6 +2483,8 @@ export async function cmdDashboard(opts) {
                     title: r.title, tags: r.tags, targetRelPath: r.targetRelPath,
                     body: (r.body || '').slice(0, 800),
                 })),
+                recordsIn: result.recordsIn,
+                errors: result.errors.slice(0, 10),
             });
         }
         catch (err) {
@@ -10210,14 +10212,25 @@ if('serviceWorker' in navigator){navigator.serviceWorker.getRegistrations().then
 
           const manifest = manifestData.manifest;
           const preview = finalData.preview || [];
+          const errorsList = finalData.errors || [];
           const manifestRows = Object.entries(manifest.formats || {})
             .map(([fmt, n]) => '<tr><td>' + escapeHtml(fmt) + '</td><td>' + n + '</td></tr>').join('');
+          const warnBlock = errorsList.length
+            ? '<div style="margin-top:10px;padding:10px;background:#fff3cd;border:1px solid #f0c36d;border-radius:6px;color:#8a5a00;font-size:13px">' +
+              '<div style="font-weight:600;margin-bottom:4px">' + errorsList.length + ' file(s) could not be ingested</div>' +
+              errorsList.map((e) => '<div style="font-family:monospace;font-size:12px">• ' + escapeHtml(e.error) + '</div>').join('') +
+              '</div>'
+            : '';
+          const emptyNote = (preview.length === 0 && !errorsList.length)
+            ? '<div style="margin-top:10px;padding:10px;background:#fff3cd;border:1px solid #f0c36d;border-radius:6px;color:#8a5a00;font-size:13px">No records extracted. The file may be empty or in an unsupported format.</div>'
+            : '';
           manifestEl.innerHTML =
             '<div class="card" style="padding:12px"><div style="font-weight:600;margin-bottom:8px">Manifest</div>' +
             '<div style="color:var(--muted);font-size:13px;margin-bottom:8px">' +
             manifest.totalFiles + ' file(s), ' + brainHumanBytes(manifest.totalBytes) +
             ' · scanned in ' + Math.floor((Date.now() - progress.startedAt) / 1000) + 's</div>' +
-            '<table class="data-table"><thead><tr><th>Format</th><th>Count</th></tr></thead><tbody>' + manifestRows + '</tbody></table></div>';
+            '<table class="data-table"><thead><tr><th>Format</th><th>Count</th></tr></thead><tbody>' + manifestRows + '</tbody></table>' +
+            warnBlock + emptyNote + '</div>';
           if (preview.length) {
             const previewHtml = preview.slice(0, 10).map((p, i) =>
               '<div class="card" style="padding:12px;margin-bottom:8px">' +
@@ -10280,14 +10293,27 @@ if('serviceWorker' in navigator){navigator.serviceWorker.getRegistrations().then
             return;
           }
           const elapsed = Math.floor((Date.now() - progress.startedAt) / 1000);
+          const errList = finalData.errors || [];
+          const headerColor = (finalData.recordsWritten > 0) ? '#4ade80' : '#e5a84a';
+          const headerIcon = (finalData.recordsWritten > 0) ? '✓' : '⚠';
+          const headerText = (finalData.recordsWritten > 0)
+            ? 'Ingestion complete'
+            : 'Ingestion finished, but nothing was written';
+          const errBlock = errList.length
+            ? '<div style="margin-top:10px;padding:10px;background:#fff3cd;border:1px solid #f0c36d;border-radius:6px;color:#8a5a00;font-size:13px">' +
+              '<div style="font-weight:600;margin-bottom:4px">' + errList.length + ' error(s)</div>' +
+              errList.map((e) => '<div style="font-family:monospace;font-size:12px">• ' + escapeHtml(e.error) + '</div>').join('') +
+              '</div>'
+            : '';
           progEl.innerHTML =
             '<div class="card" style="padding:12px">' +
-            '<div style="font-weight:600;color:#4ade80">✓ Ingestion complete · ' + elapsed + 's</div>' +
+            '<div style="font-weight:600;color:' + headerColor + '">' + headerIcon + ' ' + headerText + ' · ' + elapsed + 's</div>' +
             '<div>Records in: ' + finalData.recordsIn + '</div>' +
             '<div>Records written: ' + finalData.recordsWritten + '</div>' +
             '<div>Records skipped: ' + finalData.recordsSkipped + '</div>' +
             '<div>Records failed: ' + finalData.recordsFailed + '</div>' +
             (finalData.overviewNotePath ? '<div style="margin-top:8px">Overview note: <code>' + escapeHtml(finalData.overviewNotePath) + '</code></div>' : '') +
+            errBlock +
             '</div>';
         }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clementine-agent",
-  "version": "1.0.69",
+  "version": "1.0.70",
   "description": "Clementine — Personal AI Assistant (TypeScript)",
   "type": "module",
   "main": "dist/index.js",