mcard-js 2.1.28 → 2.1.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -0
- package/dist/FileIO.d.ts +44 -0
- package/dist/FileIO.d.ts.map +1 -0
- package/dist/FileIO.js +203 -0
- package/dist/FileIO.js.map +1 -0
- package/dist/Loader.d.ts +34 -0
- package/dist/Loader.d.ts.map +1 -0
- package/dist/Loader.js +190 -0
- package/dist/Loader.js.map +1 -0
- package/dist/index.browser.d.ts +24 -0
- package/dist/index.browser.d.ts.map +1 -0
- package/dist/index.browser.js +37 -0
- package/dist/index.browser.js.map +1 -0
- package/dist/index.d.ts +8 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -4
- package/dist/index.js.map +1 -1
- package/dist/model/ContentTypeInterpreter.d.ts +5 -0
- package/dist/model/ContentTypeInterpreter.d.ts.map +1 -1
- package/dist/model/ContentTypeInterpreter.js +7 -0
- package/dist/model/ContentTypeInterpreter.js.map +1 -1
- package/dist/model/EventProducer.js +1 -1
- package/dist/model/EventProducer.js.map +1 -1
- package/dist/model/MCard.js +2 -2
- package/dist/model/MCard.js.map +1 -1
- package/dist/model/PCard.js +2 -2
- package/dist/model/PCard.js.map +1 -1
- package/dist/model/VCard.js +2 -2
- package/dist/model/VCard.js.map +1 -1
- package/dist/model/detectors/BinarySignatureDetector.d.ts +15 -0
- package/dist/model/detectors/BinarySignatureDetector.d.ts.map +1 -0
- package/dist/model/detectors/BinarySignatureDetector.js +84 -0
- package/dist/model/detectors/BinarySignatureDetector.js.map +1 -0
- package/dist/model/detectors/registry.d.ts.map +1 -1
- package/dist/model/detectors/registry.js +4 -2
- package/dist/model/detectors/registry.js.map +1 -1
- package/dist/model/hash/HashValidator.d.ts +14 -0
- package/dist/model/hash/HashValidator.d.ts.map +1 -0
- package/dist/model/hash/HashValidator.js +57 -0
- package/dist/model/hash/HashValidator.js.map +1 -0
- package/dist/model/hash/algorithms/LocalSHA256.d.ts +2 -0
- package/dist/model/hash/algorithms/LocalSHA256.d.ts.map +1 -0
- package/dist/model/hash/algorithms/LocalSHA256.js +21 -0
- package/dist/model/hash/algorithms/LocalSHA256.js.map +1 -0
- package/dist/model/validators/BaseValidator.d.ts +21 -0
- package/dist/model/validators/BaseValidator.d.ts.map +1 -0
- package/dist/model/validators/BaseValidator.js +9 -0
- package/dist/model/validators/BaseValidator.js.map +1 -0
- package/dist/model/validators/BinaryValidator.d.ts +14 -0
- package/dist/model/validators/BinaryValidator.d.ts.map +1 -0
- package/dist/model/validators/BinaryValidator.js +127 -0
- package/dist/model/validators/BinaryValidator.js.map +1 -0
- package/dist/model/validators/TextValidator.d.ts +11 -0
- package/dist/model/validators/TextValidator.d.ts.map +1 -0
- package/dist/model/validators/TextValidator.js +84 -0
- package/dist/model/validators/TextValidator.js.map +1 -0
- package/dist/model/validators/ValidationRegistry.d.ts +15 -0
- package/dist/model/validators/ValidationRegistry.d.ts.map +1 -0
- package/dist/model/validators/ValidationRegistry.js +51 -0
- package/dist/model/validators/ValidationRegistry.js.map +1 -0
- package/dist/model/vcard_vocabulary.js +1 -1
- package/dist/model/vcard_vocabulary.js.map +1 -1
- package/dist/ptr/SandboxWorker.d.ts +30 -5
- package/dist/ptr/SandboxWorker.d.ts.map +1 -1
- package/dist/ptr/SandboxWorker.js +137 -11
- package/dist/ptr/SandboxWorker.js.map +1 -1
- package/dist/ptr/browser/network/WebSocketClient.d.ts +12 -0
- package/dist/ptr/browser/network/WebSocketClient.d.ts.map +1 -0
- package/dist/ptr/browser/network/WebSocketClient.js +47 -0
- package/dist/ptr/browser/network/WebSocketClient.js.map +1 -0
- package/dist/ptr/browser/storage/MCardStore.d.ts +14 -0
- package/dist/ptr/browser/storage/MCardStore.d.ts.map +1 -0
- package/dist/ptr/browser/storage/MCardStore.js +162 -0
- package/dist/ptr/browser/storage/MCardStore.js.map +1 -0
- package/dist/ptr/browser/worker/ServiceWorkerPTR.d.ts +12 -0
- package/dist/ptr/browser/worker/ServiceWorkerPTR.d.ts.map +1 -0
- package/dist/ptr/browser/worker/ServiceWorkerPTR.js +94 -0
- package/dist/ptr/browser/worker/ServiceWorkerPTR.js.map +1 -0
- package/dist/ptr/browser/worker/index.d.ts +2 -0
- package/dist/ptr/browser/worker/index.d.ts.map +1 -0
- package/dist/ptr/browser/worker/index.js +16 -0
- package/dist/ptr/browser/worker/index.js.map +1 -0
- package/dist/ptr/node/clm/index.d.ts +4 -4
- package/dist/ptr/node/clm/index.d.ts.map +1 -1
- package/dist/ptr/node/clm/index.js +4 -4
- package/dist/ptr/node/clm/index.js.map +1 -1
- package/dist/ptr/node/clm/runner.js +4 -4
- package/dist/ptr/node/clm/runner.js.map +1 -1
- package/dist/ptr/node/clm/utils.js +1 -1
- package/dist/ptr/node/clm/utils.js.map +1 -1
- package/dist/ptr/node/core/operations/handle.d.ts +14 -0
- package/dist/ptr/node/core/operations/handle.d.ts.map +1 -0
- package/dist/ptr/node/core/operations/handle.js +177 -0
- package/dist/ptr/node/core/operations/handle.js.map +1 -0
- package/dist/ptr/node/core/operations/index.d.ts +28 -0
- package/dist/ptr/node/core/operations/index.d.ts.map +1 -0
- package/dist/ptr/node/core/operations/index.js +94 -0
- package/dist/ptr/node/core/operations/index.js.map +1 -0
- package/dist/ptr/node/core/operations/server-scripts/websocket-server-worker.d.ts +8 -0
- package/dist/ptr/node/core/operations/server-scripts/websocket-server-worker.d.ts.map +1 -0
- package/dist/ptr/node/core/operations/server-scripts/websocket-server-worker.js +81 -0
- package/dist/ptr/node/core/operations/server-scripts/websocket-server-worker.js.map +1 -0
- package/dist/ptr/node/core/operations/static-server.d.ts +35 -0
- package/dist/ptr/node/core/operations/static-server.d.ts.map +1 -0
- package/dist/ptr/node/core/operations/static-server.js +252 -0
- package/dist/ptr/node/core/operations/static-server.js.map +1 -0
- package/dist/ptr/node/core/operations/websocket-server.d.ts +35 -0
- package/dist/ptr/node/core/operations/websocket-server.d.ts.map +1 -0
- package/dist/ptr/node/core/operations/websocket-server.js +265 -0
- package/dist/ptr/node/core/operations/websocket-server.js.map +1 -0
- package/dist/ptr/node/runtimes/loader.js +1 -1
- package/dist/ptr/node/runtimes/loader.js.map +1 -1
- package/dist/src/FileIO.d.ts +44 -0
- package/dist/src/FileIO.d.ts.map +1 -0
- package/dist/src/FileIO.js +203 -0
- package/dist/src/FileIO.js.map +1 -0
- package/dist/src/Loader.d.ts +34 -0
- package/dist/src/Loader.d.ts.map +1 -0
- package/dist/src/Loader.js +190 -0
- package/dist/src/Loader.js.map +1 -0
- package/dist/src/index.d.ts +8 -4
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +9 -4
- package/dist/src/index.js.map +1 -1
- package/dist/src/model/ContentTypeInterpreter.d.ts +5 -0
- package/dist/src/model/ContentTypeInterpreter.d.ts.map +1 -1
- package/dist/src/model/ContentTypeInterpreter.js +7 -0
- package/dist/src/model/ContentTypeInterpreter.js.map +1 -1
- package/dist/src/model/EventProducer.js +1 -1
- package/dist/src/model/EventProducer.js.map +1 -1
- package/dist/src/model/MCard.js +2 -2
- package/dist/src/model/MCard.js.map +1 -1
- package/dist/src/model/PCard.js +2 -2
- package/dist/src/model/PCard.js.map +1 -1
- package/dist/src/model/VCard.js +2 -2
- package/dist/src/model/VCard.js.map +1 -1
- package/dist/src/model/detectors/BinarySignatureDetector.d.ts +15 -0
- package/dist/src/model/detectors/BinarySignatureDetector.d.ts.map +1 -0
- package/dist/src/model/detectors/BinarySignatureDetector.js +84 -0
- package/dist/src/model/detectors/BinarySignatureDetector.js.map +1 -0
- package/dist/src/model/detectors/registry.d.ts.map +1 -1
- package/dist/src/model/detectors/registry.js +4 -2
- package/dist/src/model/detectors/registry.js.map +1 -1
- package/dist/src/model/hash/HashValidator.d.ts +14 -0
- package/dist/src/model/hash/HashValidator.d.ts.map +1 -0
- package/dist/src/model/hash/HashValidator.js +57 -0
- package/dist/src/model/hash/HashValidator.js.map +1 -0
- package/dist/src/model/hash/algorithms/LocalSHA256.d.ts +2 -0
- package/dist/src/model/hash/algorithms/LocalSHA256.d.ts.map +1 -0
- package/dist/src/model/hash/algorithms/LocalSHA256.js +21 -0
- package/dist/src/model/hash/algorithms/LocalSHA256.js.map +1 -0
- package/dist/src/model/validators/BaseValidator.d.ts +21 -0
- package/dist/src/model/validators/BaseValidator.d.ts.map +1 -0
- package/dist/src/model/validators/BaseValidator.js +9 -0
- package/dist/src/model/validators/BaseValidator.js.map +1 -0
- package/dist/src/model/validators/BinaryValidator.d.ts +14 -0
- package/dist/src/model/validators/BinaryValidator.d.ts.map +1 -0
- package/dist/src/model/validators/BinaryValidator.js +127 -0
- package/dist/src/model/validators/BinaryValidator.js.map +1 -0
- package/dist/src/model/validators/TextValidator.d.ts +11 -0
- package/dist/src/model/validators/TextValidator.d.ts.map +1 -0
- package/dist/src/model/validators/TextValidator.js +84 -0
- package/dist/src/model/validators/TextValidator.js.map +1 -0
- package/dist/src/model/validators/ValidationRegistry.d.ts +15 -0
- package/dist/src/model/validators/ValidationRegistry.d.ts.map +1 -0
- package/dist/src/model/validators/ValidationRegistry.js +51 -0
- package/dist/src/model/validators/ValidationRegistry.js.map +1 -0
- package/dist/src/model/vcard_vocabulary.js +1 -1
- package/dist/src/model/vcard_vocabulary.js.map +1 -1
- package/dist/src/ptr/browser/storage/MCardStore.d.ts +9 -4
- package/dist/src/ptr/browser/storage/MCardStore.d.ts.map +1 -1
- package/dist/src/ptr/browser/storage/MCardStore.js +143 -18
- package/dist/src/ptr/browser/storage/MCardStore.js.map +1 -1
- package/dist/src/ptr/browser/worker/ServiceWorkerPTR.d.ts.map +1 -1
- package/dist/src/ptr/browser/worker/ServiceWorkerPTR.js +7 -4
- package/dist/src/ptr/browser/worker/ServiceWorkerPTR.js.map +1 -1
- package/dist/src/ptr/node/core/operations/websocket-server.js +2 -2
- package/dist/src/ptr/node/core/operations/websocket-server.js.map +1 -1
- package/dist/src/ptr/node/runtimes/loader.js +1 -1
- package/dist/src/ptr/node/runtimes/loader.js.map +1 -1
- package/dist/storage/schema.d.ts.map +1 -1
- package/dist/storage/schema.js +5 -43
- package/dist/storage/schema.js.map +1 -1
- package/dist/storage/schema_constants.d.ts +7 -0
- package/dist/storage/schema_constants.d.ts.map +1 -0
- package/dist/storage/schema_constants.js +415 -0
- package/dist/storage/schema_constants.js.map +1 -0
- package/package.json +8 -2
package/README.md
CHANGED
|
@@ -609,6 +609,39 @@ To run integration tests against a live Ollama instance:
|
|
|
609
609
|
npx vitest tests/rag/Integration.test.ts
|
|
610
610
|
```
|
|
611
611
|
|
|
612
|
+
## Version 0.1.47 / 2.1.28 Release Notes (January 17, 2026)
|
|
613
|
+
> Directory Structure Alignment, Strict Schema, and Store Refactoring
|
|
614
|
+
|
|
615
|
+
### 📂 Directory Structure Alignment
|
|
616
|
+
* **Python Parity**: Reorganized `mcard-js` to strictly match the Python `mcard` library structure.
|
|
617
|
+
* Moved `src/hash` → `src/model/hash`.
|
|
618
|
+
* Moved `src/util/FileIO.ts` → `src/FileIO.ts`.
|
|
619
|
+
* Moved `src/util/Loader.ts` → `src/Loader.ts`.
|
|
620
|
+
* Moved Validators → `src/model/validators`.
|
|
621
|
+
* Moved Detectors → `src/model/detectors`.
|
|
622
|
+
* **Cleanup**: Removed `src/util` directory.
|
|
623
|
+
* **Duplicate Removal**: Removed duplicate `ContentTypeInterpreter` in `detectors/`.
|
|
624
|
+
|
|
625
|
+
### 💾 MCardStore Refactoring
|
|
626
|
+
* **Strict SQL Schema**: Browser `IndexedDB` schema now mirrors the canonical SQL schema exactly.
|
|
627
|
+
* Tables: `card`, `handle_registry`, `handle_history`.
|
|
628
|
+
* Indexes: `by_g_time`, `by_updated_at`, `by_previous_hash`.
|
|
629
|
+
* **API Updates**:
|
|
630
|
+
* `putCard(hash, content, g_time?)`: Replaces generic storage methods.
|
|
631
|
+
* `setHandle(handle, hash)`: Transactional handle updates with history tracking.
|
|
632
|
+
* **Verification**: Verified against the "Mapping SQL to IndexedDB" specification.
|
|
633
|
+
|
|
634
|
+
### 🔒 Standardized Hashing
|
|
635
|
+
* **SHA-256**: Implemented standard SHA-256 hashing using Web Crypto API (Browser) and Node `crypto` (Server).
|
|
636
|
+
* **Location**: `src/model/hash/algorithms/LocalSHA256.ts`.
|
|
637
|
+
* **Verification**: Validated identical hash output for identical content across Python and JS.
|
|
638
|
+
|
|
639
|
+
### 🐛 Bug Fixes
|
|
640
|
+
* **CSV Detection**: Fixed regression where short text strings were misclassified as CSV.
|
|
641
|
+
* **CLMLoader**: Fixed import path error in `ptr/node/runtimes/loader.ts`.
|
|
642
|
+
|
|
643
|
+
---
|
|
644
|
+
|
|
612
645
|
## Version 0.1.44 / 2.1.24 Release Notes (January 13, 2026)
|
|
613
646
|
> Single Source of Truth Schema Synchronization
|
|
614
647
|
|
package/dist/FileIO.d.ts
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
export interface NormalizedReadResult {
|
|
2
|
+
text: string;
|
|
3
|
+
originalSize: number;
|
|
4
|
+
originalSha256Prefix: string;
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* Stream-read bytes up to byte_cap, decode, and soft wrap.
|
|
8
|
+
*/
|
|
9
|
+
export declare function streamReadNormalizedText(filePath: string, options: {
|
|
10
|
+
byteCap: number;
|
|
11
|
+
wrapWidth: number;
|
|
12
|
+
}): Promise<NormalizedReadResult>;
|
|
13
|
+
export interface FileProcessingResult {
|
|
14
|
+
content: Uint8Array | string;
|
|
15
|
+
filename: string;
|
|
16
|
+
mimeType: string;
|
|
17
|
+
extension: string;
|
|
18
|
+
isBinary: boolean;
|
|
19
|
+
size: number;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Check if a file is likely to cause processing issues.
|
|
23
|
+
*/
|
|
24
|
+
export declare function isProblematicFile(filePath: string): Promise<boolean>;
|
|
25
|
+
/**
|
|
26
|
+
* Safely read a file with limits and timeouts.
|
|
27
|
+
*/
|
|
28
|
+
export declare function readFileSafely(filePath: string, options?: {
|
|
29
|
+
allowPathological?: boolean;
|
|
30
|
+
maxBytes?: number;
|
|
31
|
+
}): Promise<Uint8Array>;
|
|
32
|
+
/**
|
|
33
|
+
* List files in directory, filtering out problematic ones.
|
|
34
|
+
*/
|
|
35
|
+
export declare function listFiles(dirPath: string, recursive?: boolean): Promise<string[]>;
|
|
36
|
+
/**
|
|
37
|
+
* Process a file and return metadata and content.
|
|
38
|
+
*/
|
|
39
|
+
export declare function processFileContent(filePath: string, options?: {
|
|
40
|
+
forceBinary?: boolean;
|
|
41
|
+
allowPathological?: boolean;
|
|
42
|
+
maxBytes?: number;
|
|
43
|
+
}): Promise<FileProcessingResult>;
|
|
44
|
+
//# sourceMappingURL=FileIO.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"FileIO.d.ts","sourceRoot":"","sources":["../src/FileIO.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,oBAAoB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,oBAAoB,EAAE,MAAM,CAAC;CAChC;AAED;;GAEG;AACH,wBAAsB,wBAAwB,CAC1C,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAChD,OAAO,CAAC,oBAAoB,CAAC,CAqE/B;AAOD,MAAM,WAAW,oBAAoB;IACjC,OAAO,EAAE,UAAU,GAAG,MAAM,CAAC;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,OAAO,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAgC1E;AAED;;GAEG;AACH,wBAAsB,cAAc,CAChC,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE;IAAE,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAO,GACjE,OAAO,CAAC,UAAU,CAAC,CAsBrB;AAED;;GAEG;AACH,wBAAsB,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,GAAE,OAAe,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAyB9F;AAED;;GAEG;AACH,wBAAsB,kBAAkB,CACpC,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE;IAAE,WAAW,CAAC,EAAE,OAAO,CAAC;IAAC,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAO,GACxF,OAAO,CAAC,oBAAoB,CAAC,CA4B/B"}
|
package/dist/FileIO.js
ADDED
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
import * as crypto from 'crypto';
|
|
2
|
+
import * as fs from 'fs/promises';
|
|
3
|
+
import * as path from 'path';
|
|
4
|
+
import { ContentTypeInterpreter } from './model/ContentTypeInterpreter';
|
|
5
|
+
/**
|
|
6
|
+
* Stream-read bytes up to byte_cap, decode, and soft wrap.
|
|
7
|
+
*/
|
|
8
|
+
export async function streamReadNormalizedText(filePath, options) {
|
|
9
|
+
const { byteCap, wrapWidth } = options;
|
|
10
|
+
const sha = crypto.createHash('sha256');
|
|
11
|
+
let totalSize = 0;
|
|
12
|
+
let producedText = '';
|
|
13
|
+
let currentLen = 0;
|
|
14
|
+
const handle = await fs.open(filePath, 'r');
|
|
15
|
+
try {
|
|
16
|
+
const buffer = new Uint8Array(8192);
|
|
17
|
+
let remaining = byteCap;
|
|
18
|
+
// Check if TextDecoder supports replacement (it does by default fatal: false)
|
|
19
|
+
const decoder = new TextDecoder('utf-8', { fatal: false });
|
|
20
|
+
// Use position variable manually to be safe with handle.read
|
|
21
|
+
let position = 0;
|
|
22
|
+
while (remaining > 0) {
|
|
23
|
+
const { bytesRead } = await handle.read(buffer, 0, Math.min(buffer.length, remaining), position);
|
|
24
|
+
if (bytesRead === 0)
|
|
25
|
+
break;
|
|
26
|
+
position += bytesRead;
|
|
27
|
+
const chunk = buffer.subarray(0, bytesRead);
|
|
28
|
+
sha.update(chunk);
|
|
29
|
+
totalSize += bytesRead;
|
|
30
|
+
remaining -= bytesRead;
|
|
31
|
+
const s = decoder.decode(chunk, { stream: true });
|
|
32
|
+
for (const ch of s) {
|
|
33
|
+
if (ch === '\r')
|
|
34
|
+
continue;
|
|
35
|
+
producedText += ch;
|
|
36
|
+
if (ch === '\n') {
|
|
37
|
+
currentLen = 0;
|
|
38
|
+
}
|
|
39
|
+
else {
|
|
40
|
+
currentLen++;
|
|
41
|
+
if (wrapWidth > 0 && currentLen >= wrapWidth) {
|
|
42
|
+
producedText += '\n';
|
|
43
|
+
currentLen = 0;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
// Flush decoder
|
|
49
|
+
const s = decoder.decode();
|
|
50
|
+
for (const ch of s) {
|
|
51
|
+
if (ch === '\r')
|
|
52
|
+
continue;
|
|
53
|
+
producedText += ch;
|
|
54
|
+
if (ch === '\n') {
|
|
55
|
+
currentLen = 0;
|
|
56
|
+
}
|
|
57
|
+
else {
|
|
58
|
+
currentLen++;
|
|
59
|
+
if (wrapWidth > 0 && currentLen >= wrapWidth) {
|
|
60
|
+
producedText += '\n';
|
|
61
|
+
currentLen = 0;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
finally {
|
|
67
|
+
await handle.close();
|
|
68
|
+
}
|
|
69
|
+
return {
|
|
70
|
+
text: producedText,
|
|
71
|
+
originalSize: totalSize,
|
|
72
|
+
originalSha256Prefix: sha.digest('hex').substring(0, 16)
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
// Constants
|
|
76
|
+
const MAX_FILE_SIZE = 50 * 1024 * 1024; // 50MB
|
|
77
|
+
const READ_TIMEOUT_MS = 5000;
|
|
78
|
+
/**
|
|
79
|
+
* Check if a file is likely to cause processing issues.
|
|
80
|
+
*/
|
|
81
|
+
export async function isProblematicFile(filePath) {
|
|
82
|
+
try {
|
|
83
|
+
const stats = await fs.stat(filePath);
|
|
84
|
+
if (stats.size === 0)
|
|
85
|
+
return false;
|
|
86
|
+
// Skip hidden files/directories (starting with .)
|
|
87
|
+
if (path.basename(filePath).startsWith('.'))
|
|
88
|
+
return true;
|
|
89
|
+
if (stats.size > MAX_FILE_SIZE)
|
|
90
|
+
return true;
|
|
91
|
+
const ext = path.extname(filePath);
|
|
92
|
+
const isKnownType = ContentTypeInterpreter.isKnownLongLineExtension(ext);
|
|
93
|
+
if (isKnownType && stats.size > 1024 * 1024)
|
|
94
|
+
return true; // >1MB known type skip
|
|
95
|
+
// Sample content
|
|
96
|
+
const handle = await fs.open(filePath, 'r');
|
|
97
|
+
try {
|
|
98
|
+
const buffer = new Uint8Array(32 * 1024);
|
|
99
|
+
const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
|
|
100
|
+
const sample = buffer.subarray(0, bytesRead);
|
|
101
|
+
if (ContentTypeInterpreter.isUnstructuredBinary(sample))
|
|
102
|
+
return true;
|
|
103
|
+
if (ContentTypeInterpreter.hasPathologicalLines(sample, isKnownType))
|
|
104
|
+
return true;
|
|
105
|
+
}
|
|
106
|
+
finally {
|
|
107
|
+
await handle.close();
|
|
108
|
+
}
|
|
109
|
+
return false;
|
|
110
|
+
}
|
|
111
|
+
catch {
|
|
112
|
+
return true;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Safely read a file with limits and timeouts.
|
|
117
|
+
*/
|
|
118
|
+
export async function readFileSafely(filePath, options = {}) {
|
|
119
|
+
const stats = await fs.stat(filePath);
|
|
120
|
+
if (stats.size > MAX_FILE_SIZE)
|
|
121
|
+
throw new Error(`File too large: ${stats.size}`);
|
|
122
|
+
const controller = new AbortController();
|
|
123
|
+
const timeout = setTimeout(() => controller.abort(), READ_TIMEOUT_MS);
|
|
124
|
+
try {
|
|
125
|
+
const handle = await fs.open(filePath, 'r');
|
|
126
|
+
try {
|
|
127
|
+
const buffer = new Uint8Array(stats.size);
|
|
128
|
+
await handle.read(buffer, 0, stats.size, 0);
|
|
129
|
+
return buffer;
|
|
130
|
+
}
|
|
131
|
+
finally {
|
|
132
|
+
await handle.close();
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
catch (e) {
|
|
136
|
+
if (e.name === 'AbortError')
|
|
137
|
+
throw new Error(`Read timeout for ${filePath}`);
|
|
138
|
+
throw e;
|
|
139
|
+
}
|
|
140
|
+
finally {
|
|
141
|
+
clearTimeout(timeout);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* List files in directory, filtering out problematic ones.
|
|
146
|
+
*/
|
|
147
|
+
export async function listFiles(dirPath, recursive = false) {
|
|
148
|
+
let files = [];
|
|
149
|
+
try {
|
|
150
|
+
const entries = await fs.readdir(dirPath, { withFileTypes: true });
|
|
151
|
+
for (const entry of entries) {
|
|
152
|
+
const fullPath = path.join(dirPath, entry.name);
|
|
153
|
+
// Skip hidden
|
|
154
|
+
if (entry.name.startsWith('.'))
|
|
155
|
+
continue;
|
|
156
|
+
if (entry.isDirectory()) {
|
|
157
|
+
if (recursive) {
|
|
158
|
+
files = files.concat(await listFiles(fullPath, true));
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
else if (entry.isFile()) {
|
|
162
|
+
if (!(await isProblematicFile(fullPath))) {
|
|
163
|
+
files.push(fullPath);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
catch (e) {
|
|
169
|
+
console.warn(`Error listing directory ${dirPath}:`, e);
|
|
170
|
+
}
|
|
171
|
+
return files;
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* Process a file and return metadata and content.
|
|
175
|
+
*/
|
|
176
|
+
export async function processFileContent(filePath, options = {}) {
|
|
177
|
+
const rawContent = await readFileSafely(filePath, { allowPathological: options.allowPathological, maxBytes: options.maxBytes });
|
|
178
|
+
// Detect type using sample
|
|
179
|
+
const sample = rawContent.subarray(0, 1024 * 1024);
|
|
180
|
+
const detection = ContentTypeInterpreter.detectContentType(sample, path.extname(filePath));
|
|
181
|
+
let isBinary = ContentTypeInterpreter.isBinaryContent(sample, detection.mimeType);
|
|
182
|
+
if (options.forceBinary)
|
|
183
|
+
isBinary = true;
|
|
184
|
+
let content = rawContent;
|
|
185
|
+
if (!isBinary) {
|
|
186
|
+
try {
|
|
187
|
+
content = new TextDecoder('utf-8', { fatal: true }).decode(rawContent);
|
|
188
|
+
}
|
|
189
|
+
catch {
|
|
190
|
+
// Fallback to replacement
|
|
191
|
+
content = new TextDecoder('utf-8', { fatal: false }).decode(rawContent);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
return {
|
|
195
|
+
content,
|
|
196
|
+
filename: path.basename(filePath),
|
|
197
|
+
mimeType: detection.mimeType,
|
|
198
|
+
extension: detection.extension,
|
|
199
|
+
isBinary,
|
|
200
|
+
size: rawContent.length
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
//# sourceMappingURL=FileIO.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"FileIO.js","sourceRoot":"","sources":["../src/FileIO.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAElC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AAQxE;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC1C,QAAgB,EAChB,OAA+C;IAE/C,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;IACvC,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACxC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,YAAY,GAAG,EAAE,CAAC;IACtB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAC5C,IAAI,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,SAAS,GAAG,OAAO,CAAC;QACxB,8EAA8E;QAC9E,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;QAE3D,6DAA6D;QAC7D,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,OAAO,SAAS,GAAG,CAAC,EAAE,CAAC;YACnB,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,QAAQ,CAAC,CAAC;YAEjG,IAAI,SAAS,KAAK,CAAC;gBAAE,MAAM;YAE3B,QAAQ,IAAI,SAAS,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;YAC5C,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAClB,SAAS,IAAI,SAAS,CAAC;YACvB,SAAS,IAAI,SAAS,CAAC;YAEvB,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YAElD,KAAK,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;gBACjB,IAAI,EAAE,KAAK,IAAI;oBAAE,SAAS;gBAC1B,YAAY,IAAI,EAAE,CAAC;gBACnB,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;oBACd,UAAU,GAAG,CAAC,CAAC;gBACnB,CAAC;qBAAM,CAAC;oBACJ,UAAU,EAAE,CAAC;oBACb,IAAI,SAAS,GAAG,CAAC,IAAI,UAAU,IAAI,SAAS,EAAE,CAAC;wBAC3C,YAAY,IAAI,IAAI,CAAC;wBACrB,UAAU,GAAG,CAAC,CAAC;oBACnB,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;QACD,gBAAgB;QAChB,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;QAC3B,KAAK,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;YACjB,IAAI,EAAE,KAAK,IAAI;gBAAE,SAAS;YAC1B,YAAY,IAAI,EAAE,CAAC;YACnB,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;gBACd,UAAU,GAAG,CAAC,CAAC;YACnB,CAAC;iBAAM,CAAC;gBACJ,UAAU,EAAE,CAAC;gBACb,IAAI,SAAS,GAAG,CAAC,IAAI,UAAU,IAAI,SAAS,EAAE,CAAC;oBAC3C,YAAY,IAAI,IAAI,CAAC;oBACrB,UAAU,GAAG,CAAC,CAAC;gBACnB,CAAC;YACL,CAAC;QACL,CAAC;IAEL,CAAC;YAAS,CAAC;QACP,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACzB,CAAC;IAED,OAAO;QACH,IAAI,EAAE,YAAY;QAClB,YAAY,EAAE,SAAS;QACvB,oBAAoB,EAAE,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC;KAC3D,CAAC;AACN,CAAC;AAGD,YAAY;AACZ,MAAM,aAAa,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO;AAC/C,MAAM,eAAe,GAAG,IAAI,CAAC;AAW7B;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,QAAgB;IACpD,IAAI,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QAEnC,kDAAkD;QAClD,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC;QAEzD,IAAI,KAAK,CAAC,IAAI,GAAG,aAAa;YAAE,OAAO,IAAI,CAAC;QAE5C,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACnC,MAAM,WAAW,GAAG,sBAAsB,CAAC,wBAAwB,CAAC,GAAG,CAAC,CAAC;QAEzE,IAAI,WAAW,IAAI,KAAK,CAAC,IAAI,GAAG,IAAI,GAAG,IAAI;YAAE,OAAO,IAAI,CAAC,CAAC,uBAAuB;QAEjF,iBAAiB;QACjB,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAC5C,IAAI,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC;YACzC,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACrE,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;YAE7C,IAAI,sBAAsB,CAAC,oBAAoB,CAAC,MAAM,CAAC;gBAAE,OAAO,IAAI,CAAC;YACrE,IAAI,sBAAsB,CAAC,oBAAoB,CAAC,MAAM,EAAE,WAAW,CAAC;gBAAE,OAAO,IAAI,CAAC;QACtF,CAAC;gBAAS,CAAC;YACP,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACzB,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,IAAI,CAAC;IAChB,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAChC,QAAgB,EAChB,UAA8D,EAAE;IAEhE,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACtC,IAAI,KAAK,CAAC,IAAI,GAAG,aAAa;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;IAEjF,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,eAAe,CAAC,CAAC;IAEtE,IAAI,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAC5C,IAAI,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC1C,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC5C,OAAO,MAAM,CAAC;QAClB,CAAC;gBAAS,CAAC;YACP,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACzB,CAAC;IACL,CAAC;IAAC,OAAO,CAAM,EAAE,CAAC;QACd,IAAI,CAAC,CAAC,IAAI,KAAK,YAAY;YAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,QAAQ,EAAE,CAAC,CAAC;QAC7E,MAAM,CAAC,CAAC;IACZ,CAAC;YAAS,CAAC;QACP,YAAY,CAAC,OAAO,CAAC,CAAC;IAC1B,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAe,EAAE,YAAqB,KAAK;IACvE,IAAI,KAAK,GAAa,EAAE,CAAC;IACzB,IAAI,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QAEnE,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAEhD,cAAc;YACd,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;gBAAE,SAAS;YAEzC,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACtB,IAAI,SAAS,EAAE,CAAC;oBACZ,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;gBAC1D,CAAC;YACL,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;gBACxB,IAAI,CAAC,CAAC,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;oBACvC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACzB,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,OAAO,CAAC,IAAI,CAAC,2BAA2B,OAAO,GAAG,EAAE,CAAC,CAAC,CAAC;IAC3D,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACpC,QAAgB,EAChB,UAAqF,EAAE;IAEvF,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,QAAQ,EAAE,EAAE,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IAEhI,2BAA2B;IAC3B,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC,CAAC;IACnD,MAAM,SAAS,GAAG,sBAAsB,CAAC,iBAAiB,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IAE3F,IAAI,QAAQ,GAAG,sBAAsB,CAAC,eAAe,CAAC,MAAM,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;IAClF,IAAI,OAAO,CAAC,WAAW;QAAE,QAAQ,GAAG,IAAI,CAAC;IAEzC,IAAI,OAAO,GAAwB,UAAU,CAAC;IAC9C,IAAI,CAAC,QAAQ,EAAE,CAAC;QACZ,IAAI,CAAC;YACD,OAAO,GAAG,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC3E,CAAC;QAAC,MAAM,CAAC;YACL,0BAA0B;YAC1B,OAAO,GAAG,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC5E,CAAC;IACL,CAAC;IAED,OAAO;QACH,OAAO;QACP,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACjC,QAAQ,EAAE,SAAS,CAAC,QAAQ;QAC5B,SAAS,EAAE,SAAS,CAAC,SAAS;QAC9B,QAAQ;QACR,IAAI,EAAE,UAAU,CAAC,MAAM;KAC1B,CAAC;AACN,CAAC"}
|
package/dist/Loader.d.ts
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { CardCollection } from './model/CardCollection';
|
|
2
|
+
export interface FileProcessingResult {
|
|
3
|
+
hash: string;
|
|
4
|
+
contentType?: string;
|
|
5
|
+
isBinary?: boolean;
|
|
6
|
+
filename?: string;
|
|
7
|
+
size?: number;
|
|
8
|
+
filePath: string;
|
|
9
|
+
originalSize?: number;
|
|
10
|
+
originalSha256Prefix?: string;
|
|
11
|
+
metadataOnly?: boolean;
|
|
12
|
+
}
|
|
13
|
+
export declare function processAndStoreFile(filePath: string, collection: CardCollection, options?: {
|
|
14
|
+
allowProblematic?: boolean;
|
|
15
|
+
maxBytesOnProblem?: number;
|
|
16
|
+
metadataOnly?: boolean;
|
|
17
|
+
rootPath?: string;
|
|
18
|
+
}): Promise<FileProcessingResult | null>;
|
|
19
|
+
export interface LoaderMetrics {
|
|
20
|
+
filesCount: number;
|
|
21
|
+
directoriesCount: number;
|
|
22
|
+
directoryLevels: number;
|
|
23
|
+
}
|
|
24
|
+
export interface LoaderResponse {
|
|
25
|
+
metrics: LoaderMetrics;
|
|
26
|
+
results: FileProcessingResult[];
|
|
27
|
+
}
|
|
28
|
+
export declare function loadFileToCollection(targetPath: string, collection: CardCollection, options?: {
|
|
29
|
+
recursive?: boolean;
|
|
30
|
+
includeProblematic?: boolean;
|
|
31
|
+
maxBytesOnProblem?: number;
|
|
32
|
+
metadataOnly?: boolean;
|
|
33
|
+
}): Promise<LoaderResponse>;
|
|
34
|
+
//# sourceMappingURL=Loader.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Loader.d.ts","sourceRoot":"","sources":["../src/Loader.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAcxD,MAAM,WAAW,oBAAoB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,YAAY,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,wBAAsB,mBAAmB,CACrC,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,cAAc,EAC1B,OAAO,GAAE;IACL,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAChB,GACP,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,CAmItC;AAED,MAAM,WAAW,aAAa;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,cAAc;IAC3B,OAAO,EAAE,aAAa,CAAC;IACvB,OAAO,EAAE,oBAAoB,EAAE,CAAC;CACnC;AAED,wBAAsB,oBAAoB,CACtC,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,cAAc,EAC1B,OAAO,GAAE;IACL,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,YAAY,CAAC,EAAE,OAAO,CAAC;CACrB,GACP,OAAO,CAAC,cAAc,CAAC,CAkEzB"}
|
package/dist/Loader.js
ADDED
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
import * as fs from 'fs/promises';
|
|
2
|
+
import * as path from 'path';
|
|
3
|
+
import { MCard } from './model/MCard';
|
|
4
|
+
import { isProblematicFile, processFileContent, streamReadNormalizedText, listFiles } from './FileIO';
|
|
5
|
+
import { ContentTypeInterpreter } from './model/ContentTypeInterpreter';
|
|
6
|
+
// Constants replicating settings
|
|
7
|
+
const DEFAULT_MAX_PROBLEM_BYTES = 2 * 1024 * 1024; // 2MB
|
|
8
|
+
const WRAP_WIDTH_KNOWN = 1000;
|
|
9
|
+
const WRAP_WIDTH_DEFAULT = 80;
|
|
10
|
+
export async function processAndStoreFile(filePath, collection, options = {}) {
|
|
11
|
+
const { allowProblematic = false, maxBytesOnProblem = DEFAULT_MAX_PROBLEM_BYTES, metadataOnly = false, rootPath } = options;
|
|
12
|
+
try {
|
|
13
|
+
let fileInfo;
|
|
14
|
+
if (await isProblematicFile(filePath)) {
|
|
15
|
+
if (!allowProblematic) {
|
|
16
|
+
console.warn(`Skipping problematic file: ${filePath}`);
|
|
17
|
+
return null;
|
|
18
|
+
}
|
|
19
|
+
const extension = path.extname(filePath).toLowerCase();
|
|
20
|
+
const isKnownType = ContentTypeInterpreter.isKnownLongLineExtension(extension);
|
|
21
|
+
const wrapWidth = isKnownType ? WRAP_WIDTH_KNOWN : WRAP_WIDTH_DEFAULT;
|
|
22
|
+
console.warn(`Problematic file detected, processing as safe text: ${filePath}`);
|
|
23
|
+
try {
|
|
24
|
+
const streamed = await streamReadNormalizedText(filePath, {
|
|
25
|
+
byteCap: maxBytesOnProblem,
|
|
26
|
+
wrapWidth
|
|
27
|
+
});
|
|
28
|
+
fileInfo = {
|
|
29
|
+
content: streamed.text,
|
|
30
|
+
filename: path.basename(filePath),
|
|
31
|
+
mimeType: 'text/plain',
|
|
32
|
+
extension: extension,
|
|
33
|
+
isBinary: false,
|
|
34
|
+
size: streamed.text.length,
|
|
35
|
+
originalSize: streamed.originalSize,
|
|
36
|
+
originalSha256Prefix: streamed.originalSha256Prefix,
|
|
37
|
+
normalized: true,
|
|
38
|
+
wrapWidth
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
catch (e) {
|
|
42
|
+
console.warn(`Safe text processing failed, falling back to capped binary: ${filePath}`);
|
|
43
|
+
fileInfo = await processFileContent(filePath, {
|
|
44
|
+
forceBinary: true,
|
|
45
|
+
allowPathological: true,
|
|
46
|
+
maxBytes: maxBytesOnProblem
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
else {
|
|
51
|
+
console.log(`Processing file: ${filePath}`);
|
|
52
|
+
fileInfo = await processFileContent(filePath);
|
|
53
|
+
}
|
|
54
|
+
if (!fileInfo)
|
|
55
|
+
return null;
|
|
56
|
+
// Check for empty content (e.g., empty __init__.py files)
|
|
57
|
+
const content = fileInfo.content;
|
|
58
|
+
if (!content || (typeof content === 'string' && content.length === 0) ||
|
|
59
|
+
(content instanceof Uint8Array && content.length === 0)) {
|
|
60
|
+
// Empty files cannot be stored as MCards - skip gracefully
|
|
61
|
+
if (process.env.DEBUG) {
|
|
62
|
+
console.log(`Skipping empty file: ${filePath} (empty files cannot be stored as MCards)`);
|
|
63
|
+
}
|
|
64
|
+
return {
|
|
65
|
+
hash: '',
|
|
66
|
+
contentType: fileInfo.mimeType,
|
|
67
|
+
isBinary: fileInfo.isBinary,
|
|
68
|
+
filename: fileInfo.filename,
|
|
69
|
+
size: 0,
|
|
70
|
+
filePath: filePath,
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
let mcard = null;
|
|
74
|
+
const isProblematic = await isProblematicFile(filePath);
|
|
75
|
+
if (metadataOnly && isProblematic) {
|
|
76
|
+
mcard = null;
|
|
77
|
+
}
|
|
78
|
+
else {
|
|
79
|
+
mcard = await MCard.create(fileInfo.content);
|
|
80
|
+
// Handle Logic
|
|
81
|
+
const handle = path.basename(filePath);
|
|
82
|
+
try {
|
|
83
|
+
await collection.addWithHandle(mcard, handle);
|
|
84
|
+
}
|
|
85
|
+
catch (e) {
|
|
86
|
+
let registered = false;
|
|
87
|
+
if (rootPath) {
|
|
88
|
+
const relPath = path.relative(rootPath, filePath);
|
|
89
|
+
if (relPath !== handle) {
|
|
90
|
+
try {
|
|
91
|
+
await collection.addWithHandle(mcard, relPath);
|
|
92
|
+
registered = true;
|
|
93
|
+
}
|
|
94
|
+
catch (e2) {
|
|
95
|
+
// Handle name already in use - this is expected for common files like README.md, LICENSE
|
|
96
|
+
// The MCard is still stored and accessible by hash
|
|
97
|
+
if (process.env.DEBUG) {
|
|
98
|
+
console.log(`Handle name '${handle}' already in use (common for files like README.md, LICENSE). ` +
|
|
99
|
+
`MCard stored successfully with hash ${mcard.hash.slice(0, 8)}... (accessible by hash, not by handle)`);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
if (!registered) {
|
|
105
|
+
try {
|
|
106
|
+
await collection.add(mcard);
|
|
107
|
+
}
|
|
108
|
+
catch (e3) { }
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
const result = {
|
|
113
|
+
hash: mcard ? mcard.hash : 'METADATA_ONLY',
|
|
114
|
+
contentType: fileInfo.mimeType,
|
|
115
|
+
isBinary: fileInfo.isBinary,
|
|
116
|
+
filename: fileInfo.filename,
|
|
117
|
+
size: fileInfo.size,
|
|
118
|
+
filePath: filePath,
|
|
119
|
+
};
|
|
120
|
+
if (fileInfo.originalSize !== undefined)
|
|
121
|
+
result.originalSize = fileInfo.originalSize;
|
|
122
|
+
if (fileInfo.originalSha256Prefix)
|
|
123
|
+
result.originalSha256Prefix = fileInfo.originalSha256Prefix;
|
|
124
|
+
if (metadataOnly && isProblematic)
|
|
125
|
+
result.metadataOnly = true;
|
|
126
|
+
return result;
|
|
127
|
+
}
|
|
128
|
+
catch (e) {
|
|
129
|
+
console.error(`Error processing ${filePath}:`, e);
|
|
130
|
+
return null;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
export async function loadFileToCollection(targetPath, collection, options = {}) {
|
|
134
|
+
const { recursive = false, includeProblematic = false, maxBytesOnProblem = DEFAULT_MAX_PROBLEM_BYTES, metadataOnly = false } = options;
|
|
135
|
+
// Resolve absolute path
|
|
136
|
+
const resolvedPath = path.resolve(targetPath);
|
|
137
|
+
const stats = await fs.stat(resolvedPath);
|
|
138
|
+
const results = [];
|
|
139
|
+
// Determine files to process
|
|
140
|
+
let files = [];
|
|
141
|
+
let rootPath = resolvedPath;
|
|
142
|
+
if (stats.isFile()) {
|
|
143
|
+
files = [resolvedPath];
|
|
144
|
+
rootPath = path.dirname(resolvedPath);
|
|
145
|
+
}
|
|
146
|
+
else if (stats.isDirectory()) {
|
|
147
|
+
files = await listFiles(resolvedPath, recursive);
|
|
148
|
+
rootPath = resolvedPath;
|
|
149
|
+
}
|
|
150
|
+
else {
|
|
151
|
+
throw new Error(`Path ${targetPath} is not a file or directory`);
|
|
152
|
+
}
|
|
153
|
+
// Calculate Metrics
|
|
154
|
+
const uniqueDirs = new Set();
|
|
155
|
+
let maxDepth = 0;
|
|
156
|
+
for (const file of files) {
|
|
157
|
+
const dir = path.dirname(file);
|
|
158
|
+
// We track directories relative to root to verify "loaded directories"
|
|
159
|
+
// If file is directly in root, dir is root.
|
|
160
|
+
if (dir.startsWith(rootPath)) {
|
|
161
|
+
uniqueDirs.add(dir);
|
|
162
|
+
// Depth calculation
|
|
163
|
+
const rel = path.relative(rootPath, file);
|
|
164
|
+
// parts length - 1 (filename) gives folder depth.
|
|
165
|
+
// e.g. "a.txt" -> 0 depth. "sub/a.txt" -> 1 depth.
|
|
166
|
+
const parts = rel.split(path.sep);
|
|
167
|
+
const depth = parts.length - 1;
|
|
168
|
+
if (depth > maxDepth)
|
|
169
|
+
maxDepth = depth;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
const metrics = {
|
|
173
|
+
filesCount: files.length,
|
|
174
|
+
directoriesCount: uniqueDirs.size,
|
|
175
|
+
directoryLevels: maxDepth
|
|
176
|
+
};
|
|
177
|
+
console.log(`About to process ${files.length} files`);
|
|
178
|
+
for (const file of files) {
|
|
179
|
+
const result = await processAndStoreFile(file, collection, {
|
|
180
|
+
allowProblematic: includeProblematic,
|
|
181
|
+
maxBytesOnProblem,
|
|
182
|
+
metadataOnly,
|
|
183
|
+
rootPath
|
|
184
|
+
});
|
|
185
|
+
if (result)
|
|
186
|
+
results.push(result);
|
|
187
|
+
}
|
|
188
|
+
return { metrics, results };
|
|
189
|
+
}
|
|
190
|
+
//# sourceMappingURL=Loader.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Loader.js","sourceRoot":"","sources":["../src/Loader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AAEtC,OAAO,EACH,iBAAiB,EACjB,kBAAkB,EAClB,wBAAwB,EACxB,SAAS,EACZ,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AAExE,iCAAiC;AACjC,MAAM,yBAAyB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,MAAM;AACzD,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAC9B,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAc9B,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACrC,QAAgB,EAChB,UAA0B,EAC1B,UAKI,EAAE;IAEN,MAAM,EACF,gBAAgB,GAAG,KAAK,EACxB,iBAAiB,GAAG,yBAAyB,EAC7C,YAAY,GAAG,KAAK,EACpB,QAAQ,EACX,GAAG,OAAO,CAAC;IAEZ,IAAI,CAAC;QACD,IAAI,QAAa,CAAC;QAElB,IAAI,MAAM,iBAAiB,CAAC,QAAQ,CAAC,EAAE,CAAC;YACpC,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACpB,OAAO,CAAC,IAAI,CAAC,8BAA8B,QAAQ,EAAE,CAAC,CAAC;gBACvD,OAAO,IAAI,CAAC;YAChB,CAAC;YAED,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;YACvD,MAAM,WAAW,GAAG,sBAAsB,CAAC,wBAAwB,CAAC,SAAS,CAAC,CAAC;YAC/E,MAAM,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,kBAAkB,CAAC;YAEtE,OAAO,CAAC,IAAI,CAAC,uDAAuD,QAAQ,EAAE,CAAC,CAAC;YAEhF,IAAI,CAAC;gBACD,MAAM,QAAQ,GAAG,MAAM,wBAAwB,CAAC,QAAQ,EAAE;oBACtD,OAAO,EAAE,iBAAiB;oBAC1B,SAAS;iBACZ,CAAC,CAAC;gBAEH,QAAQ,GAAG;oBACP,OAAO,EAAE,QAAQ,CAAC,IAAI;oBACtB,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;oBACjC,QAAQ,EAAE,YAAY;oBACtB,SAAS,EAAE,SAAS;oBACpB,QAAQ,EAAE,KAAK;oBACf,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM;oBAC1B,YAAY,EAAE,QAAQ,CAAC,YAAY;oBACnC,oBAAoB,EAAE,QAAQ,CAAC,oBAAoB;oBACnD,UAAU,EAAE,IAAI;oBAChB,SAAS;iBACZ,CAAC;YACN,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,OAAO,CAAC,IAAI,CAAC,+DAA+D,QAAQ,EAAE,CAAC,CAAC;gBACxF,QAAQ,GAAG,MAAM,kBAAkB,CAAC,QAAQ,EAAE;oBAC1C,WAAW,EAAE,IAAI;oBACjB,iBAAiB,EAAE,IAAI;oBACvB,QAAQ,EAAE,iBAAiB;iBAC9B,CAAC,CAAC;YACP,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,OAAO,CAAC,GAAG,CAAC,oBAAoB,QAAQ,EAAE,CAAC,CAAC;YAC5C,QAAQ,GAAG,MAAM,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QAClD,CAAC;QAED,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC;QAE3B,0DAA0D;QAC1D,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC;QACjC,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC;YACjE,CAAC,OAAO,YAAY,UAAU,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;YAC1D,2DAA2D;YAC3D,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;gBACpB,OAAO,CAAC,GAAG,CAAC,wBAAwB,QAAQ,2CAA2C,CAAC,CAAC;YAC7F,CAAC;YACD,OAAO;gBACH,IAAI,EAAE,EAAE;gBACR,WAAW,EAAE,QAAQ,CAAC,QAAQ;gBAC9B,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,IAAI,EAAE,CAAC;gBACP,QAAQ,EAAE,QAAQ;aACrB,CAAC;QACN,CAAC;QAED,IAAI,KAAK,GAAiB,IAAI,CAAC;QAC/B,MAAM,aAAa,GAAG,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAExD,IAAI,YAAY,IAAI,aAAa,EAAE,CAAC;YAChC,KAAK,GAAG,IAAI,CAAC;QACjB,CAAC;aAAM,CAAC;YACJ,KAAK,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAE7C,eAAe;YACf,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACvC,IAAI,CAAC;gBACD,MAAM,UAAU,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YAClD,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,IAAI,UAAU,GAAG,KAAK,CAAC;gBACvB,IAAI,QAAQ,EAAE,CAAC;oBACX,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;oBAClD,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;wBACrB,IAAI,CAAC;4BACD,MAAM,UAAU,CAAC,aAAa,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;4BAC/C,UAAU,GAAG,IAAI,CAAC;wBACtB,CAAC;wBAAC,OAAO,EAAE,EAAE,CAAC;4BACV,yFAAyF;4BACzF,mDAAmD;4BACnD,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;gCACpB,OAAO,CAAC,GAAG,CACP,gBAAgB,MAAM,+DAA+D;oCACrF,uCAAuC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,yCAAyC,CACzG,CAAC;4BACN,CAAC;wBACL,CAAC;oBACL,CAAC;gBACL,CAAC;gBACD,IAAI,CAAC,UAAU,EAAE,CAAC;oBACd,IAAI,CAAC;wBAAC,MAAM,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;oBAAC,CAAC;oBAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;gBACvD,CAAC;YACL,CAAC;QACL,CAAC;QAED,MAAM,MAAM,GAAyB;YACjC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,eAAe;YAC1C,WAAW,EAAE,QAAQ,CAAC,QAAQ;YAC9B,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,QAAQ,EAAE,QAAQ;SACrB,CAAC;QAEF,IAAI,QAAQ,CAAC,YAAY,KAAK,SAAS;YAAE,MAAM,CAAC,YAAY,GAAG,QAAQ,CAAC,YAAY,CAAC;QACrF,IAAI,QAAQ,CAAC,oBAAoB;YAAE,MAAM,CAAC,oBAAoB,GAAG,QAAQ,CAAC,oBAAoB,CAAC;QAC/F,IAAI,YAAY,IAAI,aAAa;YAAE,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC;QAE9D,OAAO,MAAM,CAAC;IAElB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,OAAO,CAAC,KAAK,CAAC,oBAAoB,QAAQ,GAAG,EAAE,CAAC,CAAC,CAAC;QAClD,OAAO,IAAI,CAAC;IAChB,CAAC;AACL,CAAC;AAaD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACtC,UAAkB,EAClB,UAA0B,EAC1B,UAKI,EAAE;IAEN,MAAM,EACF,SAAS,GAAG,KAAK,EACjB,kBAAkB,GAAG,KAAK,EAC1B,iBAAiB,GAAG,yBAAyB,EAC7C,YAAY,GAAG,KAAK,EACvB,GAAG,OAAO,CAAC;IAEZ,wBAAwB;IACxB,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAC9C,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC1C,MAAM,OAAO,GAA2B,EAAE,CAAC;IAE3C,6BAA6B;IAC7B,IAAI,KAAK,GAAa,EAAE,CAAC;IACzB,IAAI,QAAQ,GAAG,YAAY,CAAC;IAE5B,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;QACjB,KAAK,GAAG,CAAC,YAAY,CAAC,CAAC;QACvB,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IAC1C,CAAC;SAAM,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;QAC7B,KAAK,GAAG,MAAM,SAAS,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;QACjD,QAAQ,GAAG,YAAY,CAAC;IAC5B,CAAC;SAAM,CAAC;QACJ,MAAM,IAAI,KAAK,CAAC,QAAQ,UAAU,6BAA6B,CAAC,CAAC;IACrE,CAAC;IAED,oBAAoB;IACpB,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IACrC,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC/B,uEAAuE;QACvE,4CAA4C;QAC5C,IAAI,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3B,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACpB,oBAAoB;YACpB,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAC1C,mDAAmD;YACnD,mDAAmD;YACnD,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAClC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;YAC/B,IAAI,KAAK,GAAG,QAAQ;gBAAE,QAAQ,GAAG,KAAK,CAAC;QAC3C,CAAC;IACL,CAAC;IAED,MAAM,OAAO,GAAkB;QAC3B,UAAU,EAAE,KAAK,CAAC,MAAM;QACxB,gBAAgB,EAAE,UAAU,CAAC,IAAI;QACjC,eAAe,EAAE,QAAQ;KAC5B,CAAC;IAEF,OAAO,CAAC,GAAG,CAAC,oBAAoB,KAAK,CAAC,MAAM,QAAQ,CAAC,CAAC;IAEtD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,IAAI,EAAE,UAAU,EAAE;YACvD,gBAAgB,EAAE,kBAAkB;YACpC,iBAAiB;YACjB,YAAY;YACZ,QAAQ;SACX,CAAC,CAAC;QACH,IAAI,MAAM;YAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACrC,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAChC,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
export { MCard } from './model/MCard';
|
|
2
|
+
export { GTime } from './model/GTime';
|
|
3
|
+
export { ContentHandle, validateHandle, HandleValidationError } from './model/Handle';
|
|
4
|
+
export { CardCollection } from './model/CardCollection';
|
|
5
|
+
export { ContentTypeInterpreter } from './model/ContentTypeInterpreter';
|
|
6
|
+
export { EVENT_CONSTANTS, ALGORITHM_HIERARCHY } from './model/constants';
|
|
7
|
+
export { StorageEngine, Page } from './storage/StorageAdapter';
|
|
8
|
+
export { IndexedDBEngine } from './storage/IndexedDBEngine';
|
|
9
|
+
export { SqliteWasmEngine } from './storage/SqliteWasmEngine';
|
|
10
|
+
export { HashValidator } from './model/hash/HashValidator';
|
|
11
|
+
export { computeHash } from './model/hash/algorithms/LocalSHA256';
|
|
12
|
+
export { Maybe } from './monads/Maybe';
|
|
13
|
+
export { Either } from './monads/Either';
|
|
14
|
+
export { IO } from './monads/IO';
|
|
15
|
+
export { Reader } from './monads/Reader';
|
|
16
|
+
export { Writer } from './monads/Writer';
|
|
17
|
+
export { State } from './monads/State';
|
|
18
|
+
export { LensProtocol, JsonRpcRequest, JsonRpcResponse, ErrorCodes } from './ptr/LensProtocol';
|
|
19
|
+
export * from './ptr/common_types';
|
|
20
|
+
export { ServiceWorkerPTR } from './ptr/browser/worker/ServiceWorkerPTR';
|
|
21
|
+
export { MCardStore } from './ptr/browser/storage/MCardStore';
|
|
22
|
+
export { FaroSidecar, FaroSidecarConfig } from './ptr/FaroSidecar';
|
|
23
|
+
export { ValidationRegistry, validationRegistry } from './model/validators/ValidationRegistry';
|
|
24
|
+
//# sourceMappingURL=index.browser.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.browser.d.ts","sourceRoot":"","sources":["../src/index.browser.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AACtF,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AACxE,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAGzE,OAAO,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,0BAA0B,CAAC;AAC/D,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC5D,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AAG9D,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,qCAAqC,CAAC;AAGlE,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAGvC,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAC/F,cAAc,oBAAoB,CAAC;AACnC,OAAO,EAAE,gBAAgB,EAAE,MAAM,uCAAuC,CAAC;AACzE,OAAO,EAAE,UAAU,EAAE,MAAM,kCAAkC,CAAC;AAC9D,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAQnE,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,uCAAuC,CAAC"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
// MCard JavaScript Library - Browser Entry Point
|
|
2
|
+
// Excludes Node.js specific runtimes (SqliteNodeEngine)
|
|
3
|
+
// Core model
|
|
4
|
+
export { MCard } from './model/MCard';
|
|
5
|
+
export { GTime } from './model/GTime';
|
|
6
|
+
export { ContentHandle, validateHandle, HandleValidationError } from './model/Handle';
|
|
7
|
+
export { CardCollection } from './model/CardCollection';
|
|
8
|
+
export { ContentTypeInterpreter } from './model/ContentTypeInterpreter';
|
|
9
|
+
export { EVENT_CONSTANTS, ALGORITHM_HIERARCHY } from './model/constants';
|
|
10
|
+
export { IndexedDBEngine } from './storage/IndexedDBEngine';
|
|
11
|
+
export { SqliteWasmEngine } from './storage/SqliteWasmEngine';
|
|
12
|
+
// Hash
|
|
13
|
+
export { HashValidator } from './model/hash/HashValidator';
|
|
14
|
+
export { computeHash } from './model/hash/algorithms/LocalSHA256';
|
|
15
|
+
// Monads
|
|
16
|
+
export { Maybe } from './monads/Maybe';
|
|
17
|
+
export { Either } from './monads/Either';
|
|
18
|
+
export { IO } from './monads/IO';
|
|
19
|
+
export { Reader } from './monads/Reader';
|
|
20
|
+
export { Writer } from './monads/Writer';
|
|
21
|
+
export { State } from './monads/State';
|
|
22
|
+
// PTR
|
|
23
|
+
export { LensProtocol, ErrorCodes } from './ptr/LensProtocol';
|
|
24
|
+
export * from './ptr/common_types';
|
|
25
|
+
export { ServiceWorkerPTR } from './ptr/browser/worker/ServiceWorkerPTR';
|
|
26
|
+
export { MCardStore } from './ptr/browser/storage/MCardStore';
|
|
27
|
+
export { FaroSidecar } from './ptr/FaroSidecar';
|
|
28
|
+
// export { FileIO, Loader }; // Check safety
|
|
29
|
+
// Validation
|
|
30
|
+
export { ValidationRegistry, validationRegistry } from './model/validators/ValidationRegistry';
|
|
31
|
+
// LLM (Excluded due to potential Node deps)
|
|
32
|
+
// export { LLMRuntime, promptMonad, chatMonad } from './ptr/llm/LLMRuntime';
|
|
33
|
+
// export { LLMConfig } from './ptr/llm/Config';
|
|
34
|
+
// Lambda (Excluded due to Node.js dependencies in RuntimeFactory)
|
|
35
|
+
// export * as Lambda from './ptr/lambda';
|
|
36
|
+
// export { LambdaRuntime } from './ptr/lambda/LambdaRuntime';
|
|
37
|
+
//# sourceMappingURL=index.browser.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.browser.js","sourceRoot":"","sources":["../src/index.browser.ts"],"names":[],"mappings":"AAAA,iDAAiD;AACjD,wDAAwD;AAExD,aAAa;AACb,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AACtF,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AACxE,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAIzE,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC5D,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AAE9D,OAAO;AACP,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,qCAAqC,CAAC;AAElE,SAAS;AACT,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAEvC,MAAM;AACN,OAAO,EAAE,YAAY,EAAmC,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAC/F,cAAc,oBAAoB,CAAC;AACnC,OAAO,EAAE,gBAAgB,EAAE,MAAM,uCAAuC,CAAC;AACzE,OAAO,EAAE,UAAU,EAAE,MAAM,kCAAkC,CAAC;AAC9D,OAAO,EAAE,WAAW,EAAqB,MAAM,mBAAmB,CAAC;AAKnE,6CAA6C;AAE7C,aAAa;AACb,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,uCAAuC,CAAC;AAE/F,4CAA4C;AAC5C,6EAA6E;AAC7E,gDAAgD;AAEhD,kEAAkE;AAClE,0CAA0C;AAC1C,8DAA8D"}
|
package/dist/index.d.ts
CHANGED
|
@@ -2,13 +2,13 @@ export { MCard } from './model/MCard';
|
|
|
2
2
|
export { GTime } from './model/GTime';
|
|
3
3
|
export { ContentHandle, validateHandle, HandleValidationError } from './model/Handle';
|
|
4
4
|
export { CardCollection } from './model/CardCollection';
|
|
5
|
-
export { ContentTypeInterpreter } from './model/
|
|
5
|
+
export { ContentTypeInterpreter } from './model/ContentTypeInterpreter';
|
|
6
6
|
export { EVENT_CONSTANTS, ALGORITHM_HIERARCHY } from './model/constants';
|
|
7
7
|
export { StorageEngine, Page } from './storage/StorageAdapter';
|
|
8
8
|
export { IndexedDBEngine } from './storage/IndexedDBEngine';
|
|
9
9
|
export { SqliteWasmEngine } from './storage/SqliteWasmEngine';
|
|
10
10
|
export { SqliteNodeEngine } from './storage/SqliteNodeEngine';
|
|
11
|
-
export { HashValidator } from './hash/HashValidator';
|
|
11
|
+
export { HashValidator } from './model/hash/HashValidator';
|
|
12
12
|
export { Maybe } from './monads/Maybe';
|
|
13
13
|
export { Either } from './monads/Either';
|
|
14
14
|
export { IO } from './monads/IO';
|
|
@@ -18,12 +18,16 @@ export { State } from './monads/State';
|
|
|
18
18
|
export { LensProtocol, JsonRpcRequest, JsonRpcResponse, ErrorCodes } from './ptr/LensProtocol';
|
|
19
19
|
export { SandboxWorker } from './ptr/SandboxWorker';
|
|
20
20
|
export * from './ptr/common_types';
|
|
21
|
+
export { ServiceWorkerPTR } from './ptr/browser/worker/ServiceWorkerPTR';
|
|
21
22
|
export { FaroSidecar, FaroSidecarConfig } from './ptr/FaroSidecar';
|
|
22
|
-
import * as FileIO from './
|
|
23
|
-
import * as Loader from './
|
|
23
|
+
import * as FileIO from './FileIO';
|
|
24
|
+
import * as Loader from './Loader';
|
|
24
25
|
export { FileIO, Loader };
|
|
25
26
|
export { LLMRuntime, promptMonad, chatMonad } from './ptr/llm/LLMRuntime';
|
|
26
27
|
export { LLMConfig } from './ptr/llm/Config';
|
|
27
28
|
export * as Lambda from './ptr/lambda';
|
|
28
29
|
export { LambdaRuntime } from './ptr/lambda/LambdaRuntime';
|
|
30
|
+
export { MCardStore } from './ptr/browser/storage/MCardStore';
|
|
31
|
+
export { computeHash } from './model/hash/algorithms/LocalSHA256';
|
|
32
|
+
export { ValidationRegistry, validationRegistry } from './model/validators/ValidationRegistry';
|
|
29
33
|
//# sourceMappingURL=index.d.ts.map
|