@semiont/content 0.5.6 → 0.5.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +67 -175
- package/dist/index.d.ts +12 -25
- package/dist/index.js +3 -89
- package/dist/index.js.map +1 -1
- package/package.json +5 -5
package/README.md
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
[](https://www.npmjs.com/package/@semiont/content)
|
|
7
7
|
[](https://github.com/The-AI-Alliance/semiont/blob/main/LICENSE)
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
Working-tree storage for project resources, with optional git staging, plus PDF text-layer extraction.
|
|
10
10
|
|
|
11
11
|
## Installation
|
|
12
12
|
|
|
@@ -16,223 +16,115 @@ npm install @semiont/content
|
|
|
16
16
|
|
|
17
17
|
## Architecture Context
|
|
18
18
|
|
|
19
|
-
**Infrastructure Ownership**: In production applications, the
|
|
19
|
+
**Infrastructure Ownership**: In production applications, the working tree store is **created and managed by [@semiont/make-meaning](../make-meaning/)'s `startMakeMeaning()` function**, which serves as the single orchestration point for all infrastructure components. Backend code accesses it as `knowledgeBase.content`.
|
|
20
20
|
|
|
21
|
-
The quick start example below shows direct instantiation for **testing, CLI tools, or content management scripts**.
|
|
21
|
+
The quick start example below shows direct instantiation for **testing, CLI tools, or content management scripts**.
|
|
22
22
|
|
|
23
23
|
## Quick Start
|
|
24
24
|
|
|
25
25
|
```typescript
|
|
26
|
-
import {
|
|
26
|
+
import { WorkingTreeStore, deriveStorageUri } from '@semiont/content';
|
|
27
|
+
import { SemiontProject } from '@semiont/core/node';
|
|
27
28
|
|
|
28
|
-
const
|
|
29
|
-
|
|
30
|
-
});
|
|
29
|
+
const project = new SemiontProject('/path/to/project');
|
|
30
|
+
const store = new WorkingTreeStore(project);
|
|
31
31
|
|
|
32
|
-
//
|
|
33
|
-
const
|
|
34
|
-
|
|
35
|
-
mediaType: 'text/plain',
|
|
36
|
-
language: 'en',
|
|
37
|
-
rel: 'original'
|
|
38
|
-
});
|
|
32
|
+
// Derive a stable file:// URI from a resource name
|
|
33
|
+
const uri = deriveStorageUri('My Document', 'text/markdown');
|
|
34
|
+
// => "file://my-document.md"
|
|
39
35
|
|
|
40
|
-
|
|
36
|
+
// Write content to the working tree (API/GUI/AI path)
|
|
37
|
+
const stored = await store.store(Buffer.from('# My Document\n'), uri);
|
|
38
|
+
console.log(stored.checksum); // SHA-256 hex of the content
|
|
39
|
+
console.log(stored.byteSize); // 14
|
|
41
40
|
|
|
42
|
-
//
|
|
43
|
-
const
|
|
44
|
-
console.log(retrieved.toString()); // "Hello, World!"
|
|
41
|
+
// Register a file that is already on disk (CLI path)
|
|
42
|
+
const registered = await store.register('file://docs/overview.md');
|
|
45
43
|
|
|
46
|
-
//
|
|
47
|
-
const
|
|
48
|
-
|
|
49
|
-
rel: 'copy'
|
|
50
|
-
});
|
|
44
|
+
// Read content back by URI
|
|
45
|
+
const content = await store.retrieve(uri);
|
|
46
|
+
console.log(content.toString()); // "# My Document\n"
|
|
51
47
|
|
|
52
|
-
|
|
48
|
+
// Move and remove files
|
|
49
|
+
await store.move(uri, 'file://docs/my-document.md');
|
|
50
|
+
await store.remove('file://docs/my-document.md');
|
|
53
51
|
```
|
|
54
52
|
|
|
55
|
-
##
|
|
53
|
+
## Working Tree Storage
|
|
56
54
|
|
|
57
|
-
|
|
58
|
-
- 🎯 **Automatic Deduplication** - Identical content stored once
|
|
59
|
-
- 🗂️ **Smart Sharding** - 65,536 directories for scalability
|
|
60
|
-
- 📊 **W3C Compliant** - Full representation metadata support
|
|
61
|
-
- 🏷️ **MIME Type Support** - 80+ types with proper extensions
|
|
62
|
-
- 🌍 **Multilingual** - Language and encoding metadata
|
|
55
|
+
The working tree (project root) is the source of truth for file content. Resources are identified by their `file://` URI, which is stable across content changes; moves are tracked by events.
|
|
63
56
|
|
|
64
|
-
## Documentation
|
|
65
|
-
|
|
66
|
-
- [API Reference](./docs/API.md) - Complete API documentation
|
|
67
|
-
- [Architecture](./docs/architecture.md) - Design principles
|
|
68
|
-
|
|
69
|
-
## Storage Architecture
|
|
70
|
-
|
|
71
|
-
### Content Addressing
|
|
72
|
-
|
|
73
|
-
Every piece of content is addressed by its SHA-256 checksum:
|
|
74
|
-
|
|
75
|
-
```typescript
|
|
76
|
-
const checksum = calculateChecksum(content);
|
|
77
|
-
// sha256:5aaa0b72c1f4d8e7a9f2c8b3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3
|
|
78
57
|
```
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
`.semiont/`). The `representations/` directory is committed to version control —
|
|
84
|
-
it is the durable content store for the project.
|
|
85
|
-
|
|
86
|
-
```
|
|
87
|
-
my-project/ ← basePath (project root)
|
|
88
|
-
└── representations/
|
|
89
|
-
└── {mediaType}/ # URL-encoded MIME type
|
|
90
|
-
└── {ab}/ # First 2 hex chars of checksum
|
|
91
|
-
└── {cd}/ # Next 2 hex chars (65,536 shards)
|
|
92
|
-
└── rep-{checksum}.{ext}
|
|
58
|
+
my-project/ ← project root
|
|
59
|
+
├── .semiont/ ← project config and event log
|
|
60
|
+
└── docs/
|
|
61
|
+
└── overview.md ← storageUri "file://docs/overview.md"
|
|
93
62
|
```
|
|
94
63
|
|
|
95
|
-
|
|
96
|
-
```
|
|
97
|
-
representations/text~1plain/5a/aa/rep-5aaa0b72...abc.txt
|
|
98
|
-
representations/image~1png/ff/12/rep-ff123456...def.png
|
|
99
|
-
representations/application~1json/ab/cd/rep-abcd1234...123.json
|
|
100
|
-
```
|
|
64
|
+
There are two write paths:
|
|
101
65
|
|
|
102
|
-
|
|
66
|
+
- **`store(content, storageUri)`** — write bytes to disk. Used when the file does not yet exist and the caller provides content (API/GUI/AI path).
|
|
67
|
+
- **`register(storageUri, expectedChecksum?)`** — read an existing file and record its metadata (CLI path). If `expectedChecksum` is provided and does not match, throws `ChecksumMismatchError`.
|
|
103
68
|
|
|
104
|
-
|
|
69
|
+
Both return the same metadata:
|
|
105
70
|
|
|
106
71
|
```typescript
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
72
|
+
interface StoredResource {
|
|
73
|
+
storageUri: string; // file:// URI (e.g. "file://docs/overview.md")
|
|
74
|
+
checksum: string; // SHA-256 hex of content
|
|
75
|
+
byteSize: number; // Size in bytes
|
|
76
|
+
created: string; // ISO 8601 timestamp
|
|
110
77
|
}
|
|
111
|
-
// Result: Only ONE file on disk
|
|
112
78
|
```
|
|
113
79
|
|
|
114
|
-
|
|
80
|
+
### Git Integration
|
|
115
81
|
|
|
116
|
-
|
|
82
|
+
When the project has `[git] sync = true` in `.semiont/config`, the store keeps the git index up to date automatically:
|
|
117
83
|
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
});
|
|
122
|
-
```
|
|
84
|
+
- `store()` / `register()` run `git add`
|
|
85
|
+
- `move()` runs `git mv`
|
|
86
|
+
- `remove()` runs `git rm` (or `git rm --cached` with `keepFile: true`)
|
|
123
87
|
|
|
124
|
-
|
|
88
|
+
Every method accepts `{ noGit: true }` to skip staging for a single call. Without git sync, the store falls back to plain filesystem operations.
|
|
125
89
|
|
|
126
|
-
|
|
127
|
-
const stored = await store.store(
|
|
128
|
-
content: Buffer,
|
|
129
|
-
metadata: {
|
|
130
|
-
mediaType: string; // Required: MIME type
|
|
131
|
-
filename?: string; // Optional: Original filename
|
|
132
|
-
encoding?: string; // Optional: Character encoding
|
|
133
|
-
language?: string; // Optional: ISO language code
|
|
134
|
-
rel?: string; // Optional: Relationship type
|
|
135
|
-
}
|
|
136
|
-
): Promise<StoredRepresentation>
|
|
137
|
-
```
|
|
90
|
+
## PDF Text-Layer Extraction
|
|
138
91
|
|
|
139
|
-
|
|
92
|
+
For native (non-scanned) PDFs, `extractPdfTextLayer()` extracts positioned text using [pdfjs-dist](https://www.npmjs.com/package/pdfjs-dist). It returns `null` for scanned/image-only PDFs.
|
|
140
93
|
|
|
141
94
|
```typescript
|
|
142
|
-
|
|
143
|
-
checksum: string, // SHA-256 checksum
|
|
144
|
-
mediaType: string // MIME type for path lookup
|
|
145
|
-
): Promise<Buffer>
|
|
146
|
-
```
|
|
95
|
+
import { extractPdfTextLayer, locate } from '@semiont/content';
|
|
147
96
|
|
|
148
|
-
|
|
97
|
+
const layer = await extractPdfTextLayer(pdfBytes);
|
|
98
|
+
if (layer) {
|
|
99
|
+
console.log(layer.text); // Full extracted text
|
|
100
|
+
console.log(layer.pages.length); // Page dimensions in PDF points
|
|
149
101
|
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
checksum: string; // SHA-256 hex (64 chars)
|
|
154
|
-
byteSize: number; // Content size in bytes
|
|
155
|
-
mediaType: string; // MIME type
|
|
156
|
-
created: string; // ISO 8601 timestamp
|
|
157
|
-
language?: string; // ISO language code
|
|
158
|
-
encoding?: string; // Character encoding
|
|
159
|
-
rel?: string; // Relationship type
|
|
102
|
+
// Find bounding rectangles for a span of the text (one per line)
|
|
103
|
+
const rects = locate(layer, 120, 178);
|
|
104
|
+
// => PdfCoordinate[] in PDF point space (origin: bottom-left)
|
|
160
105
|
}
|
|
161
106
|
```
|
|
162
107
|
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
The package includes 80+ MIME type mappings:
|
|
166
|
-
|
|
167
|
-
| Type | Extensions | Example |
|
|
168
|
-
|------|-----------|---------|
|
|
169
|
-
| Text | `.txt`, `.md`, `.html`, `.csv` | `text/plain` → `.txt` |
|
|
170
|
-
| Documents | `.pdf`, `.doc`, `.docx` | `application/pdf` → `.pdf` |
|
|
171
|
-
| Images | `.png`, `.jpg`, `.gif`, `.webp` | `image/png` → `.png` |
|
|
172
|
-
| Audio | `.mp3`, `.wav`, `.ogg` | `audio/mpeg` → `.mp3` |
|
|
173
|
-
| Video | `.mp4`, `.webm`, `.mov` | `video/mp4` → `.mp4` |
|
|
174
|
-
| Code | `.js`, `.ts`, `.py`, `.java` | `text/javascript` → `.js` |
|
|
175
|
-
| Data | `.json`, `.xml`, `.yaml` | `application/json` → `.json` |
|
|
108
|
+
Coordinates are in PDF point space, originating from the bottom-left of the page. The Y-flip to canvas pixels happens downstream in the browser; the server has no canvas. The `PdfCoordinate` geometry type lives in `@semiont/core` alongside the viewrect FragmentSelector codec.
|
|
176
109
|
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
## W3C Compliance
|
|
180
|
-
|
|
181
|
-
Full support for W3C representation metadata:
|
|
110
|
+
## Utilities
|
|
182
111
|
|
|
183
112
|
```typescript
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
});
|
|
190
|
-
|
|
191
|
-
// W3C-compliant metadata
|
|
192
|
-
{
|
|
193
|
-
"@id": "urn:sha256:abc123...",
|
|
194
|
-
"@type": "Representation",
|
|
195
|
-
"checksum": "sha256:abc123...",
|
|
196
|
-
"mediaType": "text/html",
|
|
197
|
-
"language": "en-US",
|
|
198
|
-
"encoding": "UTF-8",
|
|
199
|
-
"rel": "original",
|
|
200
|
-
"byteSize": 1234,
|
|
201
|
-
"created": "2024-01-01T00:00:00Z"
|
|
202
|
-
}
|
|
113
|
+
import {
|
|
114
|
+
calculateChecksum, // SHA-256 hex of a string or Buffer
|
|
115
|
+
verifyChecksum, // Compare content against an expected checksum
|
|
116
|
+
deriveStorageUri, // ("My Doc", "text/markdown") → "file://my-doc.md"
|
|
117
|
+
} from '@semiont/content';
|
|
203
118
|
```
|
|
204
119
|
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
- **Write Performance**: Limited by filesystem (typically ~100 MB/s)
|
|
209
|
-
- **Read Performance**: O(1) direct path lookup
|
|
210
|
-
- **Sharding**: 65,536 directories prevent filesystem bottlenecks
|
|
211
|
-
- **Deduplication**: 100% space savings for duplicate content
|
|
120
|
+
`deriveStorageUri` takes a `SupportedMediaType`; the media-type registry —
|
|
121
|
+
which types are admitted, their extensions, and their capabilities — lives in
|
|
122
|
+
[@semiont/core](../core/)'s `media-types.ts`. See [docs/mime-types.md](./docs/mime-types.md).
|
|
212
123
|
|
|
213
|
-
##
|
|
214
|
-
|
|
215
|
-
1. **Use Buffers**: Always pass content as Buffer for binary safety
|
|
216
|
-
2. **Specify MIME Types**: Required for proper file extensions
|
|
217
|
-
3. **Add Language Metadata**: Important for multilingual content
|
|
218
|
-
4. **Handle Missing Content**: Check existence before retrieval
|
|
219
|
-
5. **Monitor Storage**: Track disk usage and shard distribution
|
|
220
|
-
|
|
221
|
-
## Error Handling
|
|
124
|
+
## Documentation
|
|
222
125
|
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
const retrieved = await store.retrieve(checksum, mediaType);
|
|
226
|
-
} catch (error) {
|
|
227
|
-
if (error.code === 'ENOENT') {
|
|
228
|
-
// Content not found
|
|
229
|
-
} else if (error.code === 'EACCES') {
|
|
230
|
-
// Permission denied
|
|
231
|
-
} else {
|
|
232
|
-
// Other filesystem error
|
|
233
|
-
}
|
|
234
|
-
}
|
|
235
|
-
```
|
|
126
|
+
- [API Reference](./docs/API.md) - Complete API documentation
|
|
127
|
+
- [Architecture](./docs/architecture.md) - Design principles
|
|
236
128
|
|
|
237
129
|
## Development
|
|
238
130
|
|
|
@@ -252,4 +144,4 @@ npm run typecheck
|
|
|
252
144
|
|
|
253
145
|
## License
|
|
254
146
|
|
|
255
|
-
Apache-2.0
|
|
147
|
+
Apache-2.0
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { SemiontProject } from '@semiont/core/node';
|
|
2
|
-
import { Logger, PdfCoordinate } from '@semiont/core';
|
|
2
|
+
import { Logger, SupportedMediaType, PdfCoordinate } from '@semiont/core';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* WorkingTreeStore - Manages files in the project working tree
|
|
@@ -129,37 +129,24 @@ declare class ChecksumMismatchError extends Error {
|
|
|
129
129
|
}
|
|
130
130
|
|
|
131
131
|
/**
|
|
132
|
-
*
|
|
132
|
+
* Storage URI Derivation
|
|
133
133
|
*
|
|
134
|
-
*
|
|
135
|
-
*
|
|
134
|
+
* Builds the file:// URI a resource lives at in the working tree from its
|
|
135
|
+
* name and validated media type. Extensions come from the media-type
|
|
136
|
+
* registry in @semiont/core; formats are validated upstream at the
|
|
137
|
+
* create/yield boundary, so the lookup is strict — no fallback.
|
|
136
138
|
*/
|
|
139
|
+
|
|
137
140
|
/**
|
|
138
|
-
*
|
|
139
|
-
*
|
|
140
|
-
* @param mediaType - MIME type (e.g., "text/markdown")
|
|
141
|
-
* @returns File extension with leading dot (e.g., ".md") or ".dat" if unknown
|
|
141
|
+
* Derive a file:// storage URI from a resource name and media type.
|
|
142
142
|
*
|
|
143
|
-
*
|
|
144
|
-
*
|
|
145
|
-
* getExtensionForMimeType('image/png') // => '.png'
|
|
146
|
-
* getExtensionForMimeType('unknown/type') // => '.dat'
|
|
147
|
-
*/
|
|
148
|
-
declare function getExtensionForMimeType(mediaType: string): string;
|
|
149
|
-
/**
|
|
150
|
-
* Derive a file:// storage URI from a resource name and MIME type.
|
|
143
|
+
* The name is lowercased, runs of non-alphanumeric characters collapse to
|
|
144
|
+
* single hyphens, and leading/trailing hyphens are stripped.
|
|
151
145
|
*
|
|
152
146
|
* @example
|
|
153
147
|
* deriveStorageUri("My Document", "text/markdown") // => "file://my-document.md"
|
|
154
148
|
*/
|
|
155
|
-
declare function deriveStorageUri(name: string, format:
|
|
156
|
-
/**
|
|
157
|
-
* Check if a MIME type has a known extension mapping
|
|
158
|
-
*
|
|
159
|
-
* @param mediaType - MIME type to check
|
|
160
|
-
* @returns true if extension is known, false if would fallback to .dat
|
|
161
|
-
*/
|
|
162
|
-
declare function hasKnownExtension(mediaType: string): boolean;
|
|
149
|
+
declare function deriveStorageUri(name: string, format: SupportedMediaType): string;
|
|
163
150
|
|
|
164
151
|
/**
|
|
165
152
|
* Checksum utilities for content verification
|
|
@@ -243,5 +230,5 @@ declare function extractPdfTextLayer(bytes: Uint8Array | Buffer): Promise<PdfTex
|
|
|
243
230
|
*/
|
|
244
231
|
declare function locate(layer: PdfTextLayer, start: number, end: number): PdfCoordinate[];
|
|
245
232
|
|
|
246
|
-
export { ChecksumMismatchError, WorkingTreeStore, calculateChecksum, deriveStorageUri, extractPdfTextLayer,
|
|
233
|
+
export { ChecksumMismatchError, WorkingTreeStore, calculateChecksum, deriveStorageUri, extractPdfTextLayer, locate, verifyChecksum };
|
|
247
234
|
export type { PdfPageInfo, PdfTextItem, PdfTextLayer, StoredResource };
|
package/dist/index.js
CHANGED
|
@@ -195,95 +195,11 @@ The file on disk differs from the recorded checksum. Has it been modified since
|
|
|
195
195
|
actual;
|
|
196
196
|
};
|
|
197
197
|
|
|
198
|
-
// src/
|
|
199
|
-
|
|
200
|
-
// Text formats
|
|
201
|
-
"text/plain": ".txt",
|
|
202
|
-
"text/markdown": ".md",
|
|
203
|
-
"text/html": ".html",
|
|
204
|
-
"text/css": ".css",
|
|
205
|
-
"text/csv": ".csv",
|
|
206
|
-
"text/xml": ".xml",
|
|
207
|
-
// Application formats - structured data
|
|
208
|
-
"application/json": ".json",
|
|
209
|
-
"application/xml": ".xml",
|
|
210
|
-
"application/yaml": ".yaml",
|
|
211
|
-
"application/x-yaml": ".yaml",
|
|
212
|
-
// Application formats - documents
|
|
213
|
-
"application/pdf": ".pdf",
|
|
214
|
-
"application/msword": ".doc",
|
|
215
|
-
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
|
|
216
|
-
"application/vnd.ms-excel": ".xls",
|
|
217
|
-
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",
|
|
218
|
-
"application/vnd.ms-powerpoint": ".ppt",
|
|
219
|
-
"application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx",
|
|
220
|
-
// Application formats - archives
|
|
221
|
-
"application/zip": ".zip",
|
|
222
|
-
"application/gzip": ".gz",
|
|
223
|
-
"application/x-tar": ".tar",
|
|
224
|
-
"application/x-7z-compressed": ".7z",
|
|
225
|
-
// Application formats - executables/binaries
|
|
226
|
-
"application/octet-stream": ".bin",
|
|
227
|
-
"application/wasm": ".wasm",
|
|
228
|
-
// Image formats
|
|
229
|
-
"image/png": ".png",
|
|
230
|
-
"image/jpeg": ".jpg",
|
|
231
|
-
"image/gif": ".gif",
|
|
232
|
-
"image/webp": ".webp",
|
|
233
|
-
"image/svg+xml": ".svg",
|
|
234
|
-
"image/bmp": ".bmp",
|
|
235
|
-
"image/tiff": ".tiff",
|
|
236
|
-
"image/x-icon": ".ico",
|
|
237
|
-
// Audio formats
|
|
238
|
-
"audio/mpeg": ".mp3",
|
|
239
|
-
"audio/wav": ".wav",
|
|
240
|
-
"audio/ogg": ".ogg",
|
|
241
|
-
"audio/webm": ".webm",
|
|
242
|
-
"audio/aac": ".aac",
|
|
243
|
-
"audio/flac": ".flac",
|
|
244
|
-
// Video formats
|
|
245
|
-
"video/mp4": ".mp4",
|
|
246
|
-
"video/mpeg": ".mpeg",
|
|
247
|
-
"video/webm": ".webm",
|
|
248
|
-
"video/ogg": ".ogv",
|
|
249
|
-
"video/quicktime": ".mov",
|
|
250
|
-
"video/x-msvideo": ".avi",
|
|
251
|
-
// Programming languages
|
|
252
|
-
"text/javascript": ".js",
|
|
253
|
-
"application/javascript": ".js",
|
|
254
|
-
"text/x-typescript": ".ts",
|
|
255
|
-
"application/typescript": ".ts",
|
|
256
|
-
"text/x-python": ".py",
|
|
257
|
-
"text/x-java": ".java",
|
|
258
|
-
"text/x-c": ".c",
|
|
259
|
-
"text/x-c++": ".cpp",
|
|
260
|
-
"text/x-csharp": ".cs",
|
|
261
|
-
"text/x-go": ".go",
|
|
262
|
-
"text/x-rust": ".rs",
|
|
263
|
-
"text/x-ruby": ".rb",
|
|
264
|
-
"text/x-php": ".php",
|
|
265
|
-
"text/x-swift": ".swift",
|
|
266
|
-
"text/x-kotlin": ".kt",
|
|
267
|
-
"text/x-shell": ".sh",
|
|
268
|
-
// Font formats
|
|
269
|
-
"font/woff": ".woff",
|
|
270
|
-
"font/woff2": ".woff2",
|
|
271
|
-
"font/ttf": ".ttf",
|
|
272
|
-
"font/otf": ".otf"
|
|
273
|
-
};
|
|
274
|
-
function getExtensionForMimeType(mediaType) {
|
|
275
|
-
const normalized = mediaType.toLowerCase().split(";")[0].trim();
|
|
276
|
-
const extension = MIME_TO_EXTENSION[normalized];
|
|
277
|
-
return extension || ".dat";
|
|
278
|
-
}
|
|
198
|
+
// src/storage-uri.ts
|
|
199
|
+
import { MEDIA_TYPES } from "@semiont/core";
|
|
279
200
|
function deriveStorageUri(name, format) {
|
|
280
201
|
const slug = name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
|
|
281
|
-
|
|
282
|
-
return `file://${slug}${ext}`;
|
|
283
|
-
}
|
|
284
|
-
function hasKnownExtension(mediaType) {
|
|
285
|
-
const normalized = mediaType.toLowerCase().split(";")[0].trim();
|
|
286
|
-
return normalized in MIME_TO_EXTENSION;
|
|
202
|
+
return `file://${slug}${MEDIA_TYPES[format].extension}`;
|
|
287
203
|
}
|
|
288
204
|
|
|
289
205
|
// src/extract-pdf-text-layer.ts
|
|
@@ -389,8 +305,6 @@ export {
|
|
|
389
305
|
calculateChecksum,
|
|
390
306
|
deriveStorageUri,
|
|
391
307
|
extractPdfTextLayer,
|
|
392
|
-
getExtensionForMimeType,
|
|
393
|
-
hasKnownExtension,
|
|
394
308
|
locate,
|
|
395
309
|
verifyChecksum
|
|
396
310
|
};
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/working-tree-store.ts","../src/checksum.ts","../src/mime-extensions.ts","../src/extract-pdf-text-layer.ts","../src/locate.ts"],"sourcesContent":["/**\n * WorkingTreeStore - Manages files in the project working tree\n *\n * Unlike the old content-addressed RepresentationStore, this store treats\n * the working tree (project root) as the source of truth for file content.\n * Resources are identified by their file:// URI, which is stable across\n * content changes and moves (tracked by events).\n *\n * Two write paths:\n * - store(content, storageUri): Write bytes to disk (API/GUI/AI path).\n * Used when the file does not yet exist and the caller provides content.\n * - register(storageUri, expectedChecksum?): Read an existing file and\n * return its metadata (CLI path). The file is already on disk; we just\n * verify and record it. If expectedChecksum is provided, throws on mismatch.\n *\n * Storage layout:\n * {projectRoot}/{path-from-uri}\n *\n * For example, storageUri \"file://docs/overview.md\" resolves to\n * {projectRoot}/docs/overview.md\n */\n\nimport { promises as fs } from 'fs';\nimport { execFileSync } from 'child_process';\nimport path from 'path';\nimport type { SemiontProject } from '@semiont/core/node';\nimport type { Logger } from '@semiont/core';\nimport { calculateChecksum, verifyChecksum } from './checksum';\n\n/**\n * Result of store() or register()\n */\nexport interface StoredResource {\n storageUri: string; // file:// URI (e.g. \"file://docs/overview.md\")\n checksum: string; // SHA-256 hex of content\n byteSize: number; // Size in bytes\n created: string; // ISO 8601 timestamp\n}\n\n/**\n * Manages files in the project working tree\n */\nexport class WorkingTreeStore {\n private projectRoot: string;\n private gitSync: boolean;\n private logger?: Logger;\n\n constructor(project: SemiontProject, logger?: Logger) {\n this.projectRoot = project.root;\n this.gitSync = project.gitSync;\n this.logger = logger;\n }\n\n private shouldRunGit(noGit?: boolean): boolean {\n return this.gitSync && !noGit;\n }\n\n /**\n * Write content to disk at the location indicated by storageUri.\n *\n * API/GUI/AI path: caller provides bytes; file may not yet exist.\n *\n * @param content - Raw bytes to write\n * @param storageUri - file:// URI (e.g. \"file://docs/overview.md\")\n * @returns Stored resource metadata\n */\n async store(content: Buffer, storageUri: string, options?: { noGit?: boolean }): Promise<StoredResource> {\n const filePath = this.resolveUri(storageUri);\n const checksum = calculateChecksum(content);\n\n this.logger?.debug('Storing resource', { storageUri, byteSize: content.length });\n\n await fs.mkdir(path.dirname(filePath), { recursive: true });\n await fs.writeFile(filePath, content);\n\n if (this.shouldRunGit(options?.noGit)) {\n execFileSync('git', ['add', filePath], { cwd: this.projectRoot });\n }\n\n this.logger?.info('Resource stored', { storageUri, checksum, byteSize: content.length });\n\n return {\n storageUri,\n checksum,\n byteSize: content.length,\n created: new Date().toISOString(),\n };\n }\n\n /**\n * Read an existing file and return its metadata.\n *\n * CLI path: the file is already on disk. We read it to compute the checksum.\n * If expectedChecksum is provided, throws ChecksumMismatchError on mismatch.\n *\n * @param storageUri - file:// URI (e.g. \"file://docs/overview.md\")\n * @param expectedChecksum - Optional SHA-256 to verify against\n * @returns Stored resource metadata\n * @throws ChecksumMismatchError if expectedChecksum is provided and does not match\n * @throws Error if file does not exist\n */\n async register(storageUri: string, expectedChecksum?: string, options?: { noGit?: boolean }): Promise<StoredResource> {\n const filePath = this.resolveUri(storageUri);\n\n this.logger?.debug('Registering resource', { storageUri });\n\n const content = await fs.readFile(filePath);\n const checksum = calculateChecksum(content);\n\n if (expectedChecksum !== undefined && !verifyChecksum(content, expectedChecksum)) {\n throw new ChecksumMismatchError(storageUri, expectedChecksum, checksum);\n }\n\n if (this.shouldRunGit(options?.noGit)) {\n execFileSync('git', ['add', filePath], { cwd: this.projectRoot });\n }\n\n this.logger?.info('Resource registered', { storageUri, checksum, byteSize: content.length });\n\n return {\n storageUri,\n checksum,\n byteSize: content.length,\n created: new Date().toISOString(),\n };\n }\n\n /**\n * Read file content by URI.\n *\n * @param storageUri - file:// URI\n * @returns Raw bytes\n */\n async retrieve(storageUri: string): Promise<Buffer> {\n const filePath = this.resolveUri(storageUri);\n try {\n return await fs.readFile(filePath);\n } catch (error: any) {\n if (error.code === 'ENOENT') {\n throw new Error(`Resource not found: ${storageUri}`);\n }\n throw error;\n }\n }\n\n /**\n * Move a file from one URI to another.\n *\n * If .git/ exists in the project root and noGit is not set, runs `git mv`.\n * Otherwise (no .git/ or noGit: true), runs fs.rename.\n *\n * @param fromUri - Current file:// URI\n * @param toUri - New file:// URI\n * @param options.noGit - Skip git mv even if .git/ is present\n */\n async move(fromUri: string, toUri: string, options?: { noGit?: boolean }): Promise<void> {\n const fromPath = this.resolveUri(fromUri);\n const toPath = this.resolveUri(toUri);\n\n this.logger?.debug('Moving resource', { fromUri, toUri });\n\n await fs.mkdir(path.dirname(toPath), { recursive: true });\n\n if (this.shouldRunGit(options?.noGit)) {\n // git mv handles both the filesystem rename and the index update\n execFileSync('git', ['mv', fromPath, toPath], { cwd: this.projectRoot });\n } else {\n await fs.rename(fromPath, toPath);\n }\n\n this.logger?.info('Resource moved', { fromUri, toUri });\n }\n\n /**\n * Remove a file from the working tree.\n *\n * If .git/ exists and noGit is not set:\n * - keepFile false (default): runs `git rm` (removes from index and disk)\n * - keepFile true: runs `git rm --cached` (removes from index only, file stays on disk)\n * If no .git/ or noGit: true:\n * - keepFile false: runs fs.unlink\n * - keepFile true: no-op on filesystem\n *\n * @param storageUri - file:// URI\n * @param options.noGit - Skip git rm even if .git/ is present\n * @param options.keepFile - Remove from git index only; leave file on disk\n */\n async remove(storageUri: string, options?: { noGit?: boolean; keepFile?: boolean }): Promise<void> {\n const filePath = this.resolveUri(storageUri);\n const keepFile = options?.keepFile ?? false;\n\n this.logger?.debug('Removing resource', { storageUri, keepFile });\n\n const useGit = this.shouldRunGit(options?.noGit);\n\n if (useGit) {\n const gitArgs = keepFile\n ? ['rm', '--cached', filePath]\n : ['rm', filePath];\n execFileSync('git', gitArgs, { cwd: this.projectRoot });\n this.logger?.info('Resource removed', { storageUri, keepFile, git: true });\n return;\n }\n\n if (keepFile) {\n this.logger?.info('Resource removed from index (file kept on disk)', { storageUri });\n return;\n }\n\n try {\n await fs.unlink(filePath);\n this.logger?.info('Resource removed', { storageUri });\n } catch (error: any) {\n if (error.code === 'ENOENT') {\n this.logger?.warn('Resource file already absent', { storageUri });\n return;\n }\n throw error;\n }\n }\n\n /**\n * Convert a file:// URI to an absolute filesystem path.\n *\n * \"file://docs/overview.md\" → \"{projectRoot}/docs/overview.md\"\n *\n * @param storageUri - file:// URI\n * @returns Absolute path\n */\n resolveUri(storageUri: string): string {\n if (!storageUri.startsWith('file://')) {\n throw new Error(`Invalid storage URI (must start with file://): ${storageUri}`);\n }\n const relativePath = storageUri.slice('file://'.length);\n return path.join(this.projectRoot, relativePath);\n }\n}\n\n/**\n * Thrown when a registered file's checksum does not match the expected value.\n * This indicates the file on disk differs from what was recorded (e.g. modified\n * after staging, or wrong file path provided).\n */\nexport class ChecksumMismatchError extends Error {\n constructor(\n readonly storageUri: string,\n readonly expected: string,\n readonly actual: string,\n ) {\n super(\n `Checksum mismatch for ${storageUri}: expected ${expected.slice(0, 8)}... but got ${actual.slice(0, 8)}...\\n` +\n `The file on disk differs from the recorded checksum. Has it been modified since staging?`\n );\n this.name = 'ChecksumMismatchError';\n }\n}\n","/**\n * Checksum utilities for content verification\n */\n\nimport { createHash } from 'crypto';\n\n/**\n * Calculate SHA-256 checksum of content\n * @param content The content to hash\n * @returns Hex-encoded SHA-256 hash\n */\nexport function calculateChecksum(content: string | Buffer): string {\n const hash = createHash('sha256');\n hash.update(content);\n return hash.digest('hex');\n}\n\n/**\n * Verify content against a checksum\n * @param content The content to verify\n * @param checksum The expected checksum\n * @returns True if content matches checksum\n */\nexport function verifyChecksum(content: string | Buffer, checksum: string): boolean {\n return calculateChecksum(content) === checksum;\n}\n","/**\n * MIME Type to File Extension Mapping\n *\n * Maps common MIME types to their standard file extensions.\n * Used by RepresentationStore to save files with proper extensions.\n */\n\n/**\n * Comprehensive MIME type to extension mapping\n */\nconst MIME_TO_EXTENSION: Record<string, string> = {\n // Text formats\n 'text/plain': '.txt',\n 'text/markdown': '.md',\n 'text/html': '.html',\n 'text/css': '.css',\n 'text/csv': '.csv',\n 'text/xml': '.xml',\n\n // Application formats - structured data\n 'application/json': '.json',\n 'application/xml': '.xml',\n 'application/yaml': '.yaml',\n 'application/x-yaml': '.yaml',\n\n // Application formats - documents\n 'application/pdf': '.pdf',\n 'application/msword': '.doc',\n 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': '.docx',\n 'application/vnd.ms-excel': '.xls',\n 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': '.xlsx',\n 'application/vnd.ms-powerpoint': '.ppt',\n 'application/vnd.openxmlformats-officedocument.presentationml.presentation': '.pptx',\n\n // Application formats - archives\n 'application/zip': '.zip',\n 'application/gzip': '.gz',\n 'application/x-tar': '.tar',\n 'application/x-7z-compressed': '.7z',\n\n // Application formats - executables/binaries\n 'application/octet-stream': '.bin',\n 'application/wasm': '.wasm',\n\n // Image formats\n 'image/png': '.png',\n 'image/jpeg': '.jpg',\n 'image/gif': '.gif',\n 'image/webp': '.webp',\n 'image/svg+xml': '.svg',\n 'image/bmp': '.bmp',\n 'image/tiff': '.tiff',\n 'image/x-icon': '.ico',\n\n // Audio formats\n 'audio/mpeg': '.mp3',\n 'audio/wav': '.wav',\n 'audio/ogg': '.ogg',\n 'audio/webm': '.webm',\n 'audio/aac': '.aac',\n 'audio/flac': '.flac',\n\n // Video formats\n 'video/mp4': '.mp4',\n 'video/mpeg': '.mpeg',\n 'video/webm': '.webm',\n 'video/ogg': '.ogv',\n 'video/quicktime': '.mov',\n 'video/x-msvideo': '.avi',\n\n // Programming languages\n 'text/javascript': '.js',\n 'application/javascript': '.js',\n 'text/x-typescript': '.ts',\n 'application/typescript': '.ts',\n 'text/x-python': '.py',\n 'text/x-java': '.java',\n 'text/x-c': '.c',\n 'text/x-c++': '.cpp',\n 'text/x-csharp': '.cs',\n 'text/x-go': '.go',\n 'text/x-rust': '.rs',\n 'text/x-ruby': '.rb',\n 'text/x-php': '.php',\n 'text/x-swift': '.swift',\n 'text/x-kotlin': '.kt',\n 'text/x-shell': '.sh',\n\n // Font formats\n 'font/woff': '.woff',\n 'font/woff2': '.woff2',\n 'font/ttf': '.ttf',\n 'font/otf': '.otf',\n};\n\n/**\n * Get file extension for a MIME type\n *\n * @param mediaType - MIME type (e.g., \"text/markdown\")\n * @returns File extension with leading dot (e.g., \".md\") or \".dat\" if unknown\n *\n * @example\n * getExtensionForMimeType('text/markdown') // => '.md'\n * getExtensionForMimeType('image/png') // => '.png'\n * getExtensionForMimeType('unknown/type') // => '.dat'\n */\nexport function getExtensionForMimeType(mediaType: string): string {\n // Normalize MIME type (lowercase, remove parameters)\n const normalized = mediaType.toLowerCase().split(';')[0]!.trim();\n\n // Look up in mapping\n const extension = MIME_TO_EXTENSION[normalized];\n\n // Return mapped extension or fallback to .dat\n return extension || '.dat';\n}\n\n/**\n * Derive a file:// storage URI from a resource name and MIME type.\n *\n * @example\n * deriveStorageUri(\"My Document\", \"text/markdown\") // => \"file://my-document.md\"\n */\nexport function deriveStorageUri(name: string, format: string): string {\n const slug = name\n .toLowerCase()\n .replace(/[^a-z0-9]+/g, '-')\n .replace(/^-|-$/g, '');\n const ext = getExtensionForMimeType(format);\n return `file://${slug}${ext}`;\n}\n\n/**\n * Check if a MIME type has a known extension mapping\n *\n * @param mediaType - MIME type to check\n * @returns true if extension is known, false if would fallback to .dat\n */\nexport function hasKnownExtension(mediaType: string): boolean {\n const normalized = mediaType.toLowerCase().split(';')[0]!.trim();\n return normalized in MIME_TO_EXTENSION;\n}\n","/**\n * PDF Text Layer Extraction\n *\n * Extracts positioned text from native, non-scanned PDFs using pdfjs-dist.\n * Returns null for scanned/image-only PDFs (no text items).\n *\n * Coordinates are in PDF point space, originating from the bottom-left.\n * The Y-flip to canvas pixels happens downstream.\n */\n\nimport * as pdfjs from 'pdfjs-dist/legacy/build/pdf.mjs';\nimport type { PdfTextLayer, PdfPageInfo, PdfTextItem } from './pdf-text-layer';\n\nexport async function extractPdfTextLayer(\n bytes: Uint8Array | Buffer\n): Promise<PdfTextLayer | null> {\n // pdf.js v5 removed the isEvalSupported option; this path only calls\n // getTextContent (no rendering / no PDF functions).\n const doc = await pdfjs.getDocument({ data: bytes }).promise;\n\n try {\n const pages: PdfPageInfo[] = [];\n const items: PdfTextItem[] = [];\n let text = '';\n let hasAnyTextItems = false;\n\n for (let pageNum = 1; pageNum <= doc.numPages; pageNum++) {\n const page = await doc.getPage(pageNum);\n const viewport = page.getViewport({ scale: 1.0 });\n const content = await page.getTextContent(); // all text items on the page\n\n pages.push({\n pageNumber: pageNum,\n widthPt: viewport.width,\n heightPt: viewport.height,\n });\n\n for (const item of content.items) {\n if (!('str' in item)) continue; // skip marked-content items (no text)\n\n if (item.str.trim()) {\n hasAnyTextItems = true;\n const start = text.length;\n text += item.str;\n const end = text.length; // range covers only this run's own chars\n\n const [, , , , x, y] = item.transform as number[];\n\n items.push({\n start,\n end,\n page: pageNum,\n x,\n y,\n width: item.width,\n height: item.height,\n });\n\n // Separator AFTER recording the run, so its [start, end) never\n // includes it. pdf.js flags the last run on a line with hasEOL —\n // newline there, space between words otherwise, so reading-order\n // lines don't glue (e.g. \"textsecond\").\n text += item.hasEOL ? '\\n' : ' ';\n } else if (item.hasEOL) {\n // Standalone end-of-line marker (empty/whitespace str): keep the\n // line break without letting whitespace-only runs add stray spaces.\n text += '\\n';\n }\n }\n\n text += '\\n'; // page break\n }\n\n if (!hasAnyTextItems) return null;\n\n return { pages, text, items };\n } finally {\n // Release the pdf.js document — Phase 2 runs this in a long-lived worker pool.\n await doc.destroy();\n }\n}\n","import type { PdfCoordinate } from '@semiont/core';\nimport type { PdfTextLayer, PdfTextItem } from './pdf-text-layer';\n\n/**\n * Items whose baseline Y is within this many PDF points are treated as being on\n * the same line. Tuned for ~12pt body text; revisit for documents with large or\n * variable font sizes (Phase 4 / #738).\n */\nconst SAME_LINE_THRESHOLD_PT = 2;\n\n/**\n * Locates bounding rectangles for a span of text in a PdfTextLayer\n * (single-line or multi-line).\n * \n * Finds all overlapping items [start, end), groups them by page and line,\n * and records one bounding rectangle per line as a PdfCoordinate.\n * \n * Returns array of PdfCoordinate, one per line of text covered by the span.\n * Returns empty array if no items overlap the span.\n */\nexport function locate(\n layer: PdfTextLayer,\n start: number,\n end: number\n): PdfCoordinate[] {\n const overlap: PdfTextItem[] = layer.items.filter(\n item => item.start < end && item.end > start\n );\n if (overlap.length === 0) return [];\n\n const pages: Map<number, PdfTextItem[]> = groupItemsByPage(overlap);\n const rects: PdfCoordinate[] = [];\n\n // for each page, group items into lines and compute one rectangle per line\n for (const [page, pageItems] of pages) {\n const lines = groupItemsByLine(pageItems, SAME_LINE_THRESHOLD_PT);\n // Compute one bounding rectangle per line and add it to rects\n for (const lineItems of lines) {\n const x = Math.min(...lineItems.map(i => i.x));\n const right = Math.max(...lineItems.map(i => i.x + i.width));\n const y = Math.min(...lineItems.map(i => i.y));\n const top = Math.max(...lineItems.map(i => i.y + i.height));\n rects.push({page, x, y, width: right - x, height: top - y});\n }\n }\n return rects;\n}\n\nfunction groupItemsByPage(items: PdfTextItem[]): Map<number, PdfTextItem[]> {\n const map = new Map<number, PdfTextItem[]>();\n for (const item of items) {\n const existing = map.get(item.page);\n if (existing) {\n existing.push(item);\n } else {\n map.set(item.page, [item]);\n }\n }\n return map;\n}\n\n\n/**\n * Sorts text items into lines when their y coordinates are\n * within `sameLineThreshold` points of each other.\n * Sorted top-to-bottom (descending y in PDF space), then left-to-right.\n * \n * Returns 2D array: \n * Outer array = list of lines\n * Inner array = list of items on that line\n*/\nfunction groupItemsByLine(items: PdfTextItem[], sameLineThreshold: number): PdfTextItem[][] {\n // Sort top-to-bottom by y; if y is equal, sort left-to-right by x\n const sorted = [...items].sort((a, b) => b.y - a.y || a.x - b.x);\n const lines: PdfTextItem[][] = [];\n let currentLine: PdfTextItem[] = [];\n\n for (const item of sorted) {\n if (currentLine.length === 0 || Math.abs(item.y - currentLine[0].y) <= sameLineThreshold) {\n currentLine.push(item);\n } else {\n lines.push(currentLine);\n currentLine = [item];\n }\n }\n if (currentLine.length > 0) lines.push(currentLine);\n return lines;\n}\n"],"mappings":";AAsBA,SAAS,YAAY,UAAU;AAC/B,SAAS,oBAAoB;AAC7B,OAAO,UAAU;;;ACpBjB,SAAS,kBAAkB;AAOpB,SAAS,kBAAkB,SAAkC;AAClE,QAAM,OAAO,WAAW,QAAQ;AAChC,OAAK,OAAO,OAAO;AACnB,SAAO,KAAK,OAAO,KAAK;AAC1B;AAQO,SAAS,eAAe,SAA0B,UAA2B;AAClF,SAAO,kBAAkB,OAAO,MAAM;AACxC;;;ADiBO,IAAM,mBAAN,MAAuB;AAAA,EACpB;AAAA,EACA;AAAA,EACA;AAAA,EAER,YAAY,SAAyB,QAAiB;AACpD,SAAK,cAAc,QAAQ;AAC3B,SAAK,UAAU,QAAQ;AACvB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEQ,aAAa,OAA0B;AAC7C,WAAO,KAAK,WAAW,CAAC;AAAA,EAC1B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWA,MAAM,MAAM,SAAiB,YAAoB,SAAwD;AACvG,UAAM,WAAW,KAAK,WAAW,UAAU;AAC3C,UAAM,WAAW,kBAAkB,OAAO;AAE1C,SAAK,QAAQ,MAAM,oBAAoB,EAAE,YAAY,UAAU,QAAQ,OAAO,CAAC;AAE/E,UAAM,GAAG,MAAM,KAAK,QAAQ,QAAQ,GAAG,EAAE,WAAW,KAAK,CAAC;AAC1D,UAAM,GAAG,UAAU,UAAU,OAAO;AAEpC,QAAI,KAAK,aAAa,SAAS,KAAK,GAAG;AACrC,mBAAa,OAAO,CAAC,OAAO,QAAQ,GAAG,EAAE,KAAK,KAAK,YAAY,CAAC;AAAA,IAClE;AAEA,SAAK,QAAQ,KAAK,mBAAmB,EAAE,YAAY,UAAU,UAAU,QAAQ,OAAO,CAAC;AAEvF,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA,UAAU,QAAQ;AAAA,MAClB,UAAS,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAcA,MAAM,SAAS,YAAoB,kBAA2B,SAAwD;AACpH,UAAM,WAAW,KAAK,WAAW,UAAU;AAE3C,SAAK,QAAQ,MAAM,wBAAwB,EAAE,WAAW,CAAC;AAEzD,UAAM,UAAU,MAAM,GAAG,SAAS,QAAQ;AAC1C,UAAM,WAAW,kBAAkB,OAAO;AAE1C,QAAI,qBAAqB,UAAa,CAAC,eAAe,SAAS,gBAAgB,GAAG;AAChF,YAAM,IAAI,sBAAsB,YAAY,kBAAkB,QAAQ;AAAA,IACxE;AAEA,QAAI,KAAK,aAAa,SAAS,KAAK,GAAG;AACrC,mBAAa,OAAO,CAAC,OAAO,QAAQ,GAAG,EAAE,KAAK,KAAK,YAAY,CAAC;AAAA,IAClE;AAEA,SAAK,QAAQ,KAAK,uBAAuB,EAAE,YAAY,UAAU,UAAU,QAAQ,OAAO,CAAC;AAE3F,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA,UAAU,QAAQ;AAAA,MAClB,UAAS,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAM,SAAS,YAAqC;AAClD,UAAM,WAAW,KAAK,WAAW,UAAU;AAC3C,QAAI;AACF,aAAO,MAAM,GAAG,SAAS,QAAQ;AAAA,IACnC,SAAS,OAAY;AACnB,UAAI,MAAM,SAAS,UAAU;AAC3B,cAAM,IAAI,MAAM,uBAAuB,UAAU,EAAE;AAAA,MACrD;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,MAAM,KAAK,SAAiB,OAAe,SAA8C;AACvF,UAAM,WAAW,KAAK,WAAW,OAAO;AACxC,UAAM,SAAS,KAAK,WAAW,KAAK;AAEpC,SAAK,QAAQ,MAAM,mBAAmB,EAAE,SAAS,MAAM,CAAC;AAExD,UAAM,GAAG,MAAM,KAAK,QAAQ,MAAM,GAAG,EAAE,WAAW,KAAK,CAAC;AAExD,QAAI,KAAK,aAAa,SAAS,KAAK,GAAG;AAErC,mBAAa,OAAO,CAAC,MAAM,UAAU,MAAM,GAAG,EAAE,KAAK,KAAK,YAAY,CAAC;AAAA,IACzE,OAAO;AACL,YAAM,GAAG,OAAO,UAAU,MAAM;AAAA,IAClC;AAEA,SAAK,QAAQ,KAAK,kBAAkB,EAAE,SAAS,MAAM,CAAC;AAAA,EACxD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAgBA,MAAM,OAAO,YAAoB,SAAkE;AACjG,UAAM,WAAW,KAAK,WAAW,UAAU;AAC3C,UAAM,WAAW,SAAS,YAAY;AAEtC,SAAK,QAAQ,MAAM,qBAAqB,EAAE,YAAY,SAAS,CAAC;AAEhE,UAAM,SAAS,KAAK,aAAa,SAAS,KAAK;AAE/C,QAAI,QAAQ;AACV,YAAM,UAAU,WACZ,CAAC,MAAM,YAAY,QAAQ,IAC3B,CAAC,MAAM,QAAQ;AACnB,mBAAa,OAAO,SAAS,EAAE,KAAK,KAAK,YAAY,CAAC;AACtD,WAAK,QAAQ,KAAK,oBAAoB,EAAE,YAAY,UAAU,KAAK,KAAK,CAAC;AACzE;AAAA,IACF;AAEA,QAAI,UAAU;AACZ,WAAK,QAAQ,KAAK,mDAAmD,EAAE,WAAW,CAAC;AACnF;AAAA,IACF;AAEA,QAAI;AACF,YAAM,GAAG,OAAO,QAAQ;AACxB,WAAK,QAAQ,KAAK,oBAAoB,EAAE,WAAW,CAAC;AAAA,IACtD,SAAS,OAAY;AACnB,UAAI,MAAM,SAAS,UAAU;AAC3B,aAAK,QAAQ,KAAK,gCAAgC,EAAE,WAAW,CAAC;AAChE;AAAA,MACF;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,WAAW,YAA4B;AACrC,QAAI,CAAC,WAAW,WAAW,SAAS,GAAG;AACrC,YAAM,IAAI,MAAM,kDAAkD,UAAU,EAAE;AAAA,IAChF;AACA,UAAM,eAAe,WAAW,MAAM,UAAU,MAAM;AACtD,WAAO,KAAK,KAAK,KAAK,aAAa,YAAY;AAAA,EACjD;AACF;AAOO,IAAM,wBAAN,cAAoC,MAAM;AAAA,EAC/C,YACW,YACA,UACA,QACT;AACA;AAAA,MACE,yBAAyB,UAAU,cAAc,SAAS,MAAM,GAAG,CAAC,CAAC,eAAe,OAAO,MAAM,GAAG,CAAC,CAAC;AAAA;AAAA,IAExG;AAPS;AACA;AACA;AAMT,SAAK,OAAO;AAAA,EACd;AAAA,EATW;AAAA,EACA;AAAA,EACA;AAQb;;;AErPA,IAAM,oBAA4C;AAAA;AAAA,EAEhD,cAAc;AAAA,EACd,iBAAiB;AAAA,EACjB,aAAa;AAAA,EACb,YAAY;AAAA,EACZ,YAAY;AAAA,EACZ,YAAY;AAAA;AAAA,EAGZ,oBAAoB;AAAA,EACpB,mBAAmB;AAAA,EACnB,oBAAoB;AAAA,EACpB,sBAAsB;AAAA;AAAA,EAGtB,mBAAmB;AAAA,EACnB,sBAAsB;AAAA,EACtB,2EAA2E;AAAA,EAC3E,4BAA4B;AAAA,EAC5B,qEAAqE;AAAA,EACrE,iCAAiC;AAAA,EACjC,6EAA6E;AAAA;AAAA,EAG7E,mBAAmB;AAAA,EACnB,oBAAoB;AAAA,EACpB,qBAAqB;AAAA,EACrB,+BAA+B;AAAA;AAAA,EAG/B,4BAA4B;AAAA,EAC5B,oBAAoB;AAAA;AAAA,EAGpB,aAAa;AAAA,EACb,cAAc;AAAA,EACd,aAAa;AAAA,EACb,cAAc;AAAA,EACd,iBAAiB;AAAA,EACjB,aAAa;AAAA,EACb,cAAc;AAAA,EACd,gBAAgB;AAAA;AAAA,EAGhB,cAAc;AAAA,EACd,aAAa;AAAA,EACb,aAAa;AAAA,EACb,cAAc;AAAA,EACd,aAAa;AAAA,EACb,cAAc;AAAA;AAAA,EAGd,aAAa;AAAA,EACb,cAAc;AAAA,EACd,cAAc;AAAA,EACd,aAAa;AAAA,EACb,mBAAmB;AAAA,EACnB,mBAAmB;AAAA;AAAA,EAGnB,mBAAmB;AAAA,EACnB,0BAA0B;AAAA,EAC1B,qBAAqB;AAAA,EACrB,0BAA0B;AAAA,EAC1B,iBAAiB;AAAA,EACjB,eAAe;AAAA,EACf,YAAY;AAAA,EACZ,cAAc;AAAA,EACd,iBAAiB;AAAA,EACjB,aAAa;AAAA,EACb,eAAe;AAAA,EACf,eAAe;AAAA,EACf,cAAc;AAAA,EACd,gBAAgB;AAAA,EAChB,iBAAiB;AAAA,EACjB,gBAAgB;AAAA;AAAA,EAGhB,aAAa;AAAA,EACb,cAAc;AAAA,EACd,YAAY;AAAA,EACZ,YAAY;AACd;AAaO,SAAS,wBAAwB,WAA2B;AAEjE,QAAM,aAAa,UAAU,YAAY,EAAE,MAAM,GAAG,EAAE,CAAC,EAAG,KAAK;AAG/D,QAAM,YAAY,kBAAkB,UAAU;AAG9C,SAAO,aAAa;AACtB;AAQO,SAAS,iBAAiB,MAAc,QAAwB;AACrE,QAAM,OAAO,KACV,YAAY,EACZ,QAAQ,eAAe,GAAG,EAC1B,QAAQ,UAAU,EAAE;AACvB,QAAM,MAAM,wBAAwB,MAAM;AAC1C,SAAO,UAAU,IAAI,GAAG,GAAG;AAC7B;AAQO,SAAS,kBAAkB,WAA4B;AAC5D,QAAM,aAAa,UAAU,YAAY,EAAE,MAAM,GAAG,EAAE,CAAC,EAAG,KAAK;AAC/D,SAAO,cAAc;AACvB;;;ACnIA,YAAY,WAAW;AAGvB,eAAsB,oBAClB,OAC4B;AAG5B,QAAM,MAAM,MAAY,kBAAY,EAAE,MAAM,MAAM,CAAC,EAAE;AAErD,MAAI;AACA,UAAM,QAAuB,CAAC;AAC9B,UAAM,QAAuB,CAAC;AAC9B,QAAI,OAAO;AACX,QAAI,kBAAkB;AAEtB,aAAS,UAAU,GAAG,WAAW,IAAI,UAAU,WAAW;AACtD,YAAM,OAAO,MAAM,IAAI,QAAQ,OAAO;AACtC,YAAM,WAAW,KAAK,YAAY,EAAE,OAAO,EAAI,CAAC;AAChD,YAAM,UAAU,MAAM,KAAK,eAAe;AAE1C,YAAM,KAAK;AAAA,QACP,YAAY;AAAA,QACZ,SAAS,SAAS;AAAA,QAClB,UAAU,SAAS;AAAA,MACvB,CAAC;AAED,iBAAW,QAAQ,QAAQ,OAAO;AAC9B,YAAI,EAAE,SAAS,MAAO;AAEtB,YAAI,KAAK,IAAI,KAAK,GAAG;AACjB,4BAAkB;AAClB,gBAAM,QAAQ,KAAK;AACnB,kBAAQ,KAAK;AACb,gBAAM,MAAM,KAAK;AAEjB,gBAAM,CAAC,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,IAAI,KAAK;AAE5B,gBAAM,KAAK;AAAA,YACP;AAAA,YACA;AAAA,YACA,MAAM;AAAA,YACN;AAAA,YACA;AAAA,YACA,OAAO,KAAK;AAAA,YACZ,QAAQ,KAAK;AAAA,UACjB,CAAC;AAMD,kBAAQ,KAAK,SAAS,OAAO;AAAA,QACjC,WAAW,KAAK,QAAQ;AAGpB,kBAAQ;AAAA,QACZ;AAAA,MACJ;AAEA,cAAQ;AAAA,IACZ;AAEA,QAAI,CAAC,gBAAiB,QAAO;AAE7B,WAAO,EAAE,OAAO,MAAM,MAAM;AAAA,EAChC,UAAE;AAEE,UAAM,IAAI,QAAQ;AAAA,EACtB;AACJ;;;ACxEA,IAAM,yBAAyB;AAYxB,SAAS,OACZ,OACA,OACA,KACe;AACf,QAAM,UAAyB,MAAM,MAAM;AAAA,IACvC,UAAQ,KAAK,QAAQ,OAAO,KAAK,MAAM;AAAA,EAC3C;AACA,MAAI,QAAQ,WAAW,EAAG,QAAO,CAAC;AAElC,QAAM,QAAoC,iBAAiB,OAAO;AAClE,QAAM,QAAyB,CAAC;AAGhC,aAAW,CAAC,MAAM,SAAS,KAAK,OAAO;AACnC,UAAM,QAAQ,iBAAiB,WAAW,sBAAsB;AAEhE,eAAW,aAAa,OAAO;AAC3B,YAAM,IAAI,KAAK,IAAI,GAAG,UAAU,IAAI,OAAK,EAAE,CAAC,CAAC;AAC7C,YAAM,QAAQ,KAAK,IAAI,GAAG,UAAU,IAAI,OAAK,EAAE,IAAI,EAAE,KAAK,CAAC;AAC3D,YAAM,IAAI,KAAK,IAAI,GAAG,UAAU,IAAI,OAAK,EAAE,CAAC,CAAC;AAC7C,YAAM,MAAM,KAAK,IAAI,GAAG,UAAU,IAAI,OAAK,EAAE,IAAI,EAAE,MAAM,CAAC;AAC1D,YAAM,KAAK,EAAC,MAAM,GAAG,GAAG,OAAO,QAAQ,GAAG,QAAQ,MAAM,EAAC,CAAC;AAAA,IAC9D;AAAA,EACJ;AACA,SAAO;AACX;AAEA,SAAS,iBAAiB,OAAkD;AACxE,QAAM,MAAM,oBAAI,IAA2B;AAC3C,aAAW,QAAQ,OAAO;AACtB,UAAM,WAAW,IAAI,IAAI,KAAK,IAAI;AAClC,QAAI,UAAU;AACV,eAAS,KAAK,IAAI;AAAA,IACtB,OAAO;AACH,UAAI,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC;AAAA,IAC7B;AAAA,EACJ;AACA,SAAO;AACX;AAYA,SAAS,iBAAiB,OAAsB,mBAA4C;AAExF,QAAM,SAAS,CAAC,GAAG,KAAK,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AAC/D,QAAM,QAAyB,CAAC;AAChC,MAAI,cAA6B,CAAC;AAElC,aAAW,QAAQ,QAAQ;AACvB,QAAI,YAAY,WAAW,KAAK,KAAK,IAAI,KAAK,IAAI,YAAY,CAAC,EAAE,CAAC,KAAK,mBAAmB;AACtF,kBAAY,KAAK,IAAI;AAAA,IACzB,OAAO;AACH,YAAM,KAAK,WAAW;AACtB,oBAAc,CAAC,IAAI;AAAA,IACvB;AAAA,EACJ;AACA,MAAI,YAAY,SAAS,EAAG,OAAM,KAAK,WAAW;AAClD,SAAO;AACX;","names":[]}
|
|
1
|
+
{"version":3,"sources":["../src/working-tree-store.ts","../src/checksum.ts","../src/storage-uri.ts","../src/extract-pdf-text-layer.ts","../src/locate.ts"],"sourcesContent":["/**\n * WorkingTreeStore - Manages files in the project working tree\n *\n * Unlike the old content-addressed RepresentationStore, this store treats\n * the working tree (project root) as the source of truth for file content.\n * Resources are identified by their file:// URI, which is stable across\n * content changes and moves (tracked by events).\n *\n * Two write paths:\n * - store(content, storageUri): Write bytes to disk (API/GUI/AI path).\n * Used when the file does not yet exist and the caller provides content.\n * - register(storageUri, expectedChecksum?): Read an existing file and\n * return its metadata (CLI path). The file is already on disk; we just\n * verify and record it. If expectedChecksum is provided, throws on mismatch.\n *\n * Storage layout:\n * {projectRoot}/{path-from-uri}\n *\n * For example, storageUri \"file://docs/overview.md\" resolves to\n * {projectRoot}/docs/overview.md\n */\n\nimport { promises as fs } from 'fs';\nimport { execFileSync } from 'child_process';\nimport path from 'path';\nimport type { SemiontProject } from '@semiont/core/node';\nimport type { Logger } from '@semiont/core';\nimport { calculateChecksum, verifyChecksum } from './checksum';\n\n/**\n * Result of store() or register()\n */\nexport interface StoredResource {\n storageUri: string; // file:// URI (e.g. \"file://docs/overview.md\")\n checksum: string; // SHA-256 hex of content\n byteSize: number; // Size in bytes\n created: string; // ISO 8601 timestamp\n}\n\n/**\n * Manages files in the project working tree\n */\nexport class WorkingTreeStore {\n private projectRoot: string;\n private gitSync: boolean;\n private logger?: Logger;\n\n constructor(project: SemiontProject, logger?: Logger) {\n this.projectRoot = project.root;\n this.gitSync = project.gitSync;\n this.logger = logger;\n }\n\n private shouldRunGit(noGit?: boolean): boolean {\n return this.gitSync && !noGit;\n }\n\n /**\n * Write content to disk at the location indicated by storageUri.\n *\n * API/GUI/AI path: caller provides bytes; file may not yet exist.\n *\n * @param content - Raw bytes to write\n * @param storageUri - file:// URI (e.g. \"file://docs/overview.md\")\n * @returns Stored resource metadata\n */\n async store(content: Buffer, storageUri: string, options?: { noGit?: boolean }): Promise<StoredResource> {\n const filePath = this.resolveUri(storageUri);\n const checksum = calculateChecksum(content);\n\n this.logger?.debug('Storing resource', { storageUri, byteSize: content.length });\n\n await fs.mkdir(path.dirname(filePath), { recursive: true });\n await fs.writeFile(filePath, content);\n\n if (this.shouldRunGit(options?.noGit)) {\n execFileSync('git', ['add', filePath], { cwd: this.projectRoot });\n }\n\n this.logger?.info('Resource stored', { storageUri, checksum, byteSize: content.length });\n\n return {\n storageUri,\n checksum,\n byteSize: content.length,\n created: new Date().toISOString(),\n };\n }\n\n /**\n * Read an existing file and return its metadata.\n *\n * CLI path: the file is already on disk. We read it to compute the checksum.\n * If expectedChecksum is provided, throws ChecksumMismatchError on mismatch.\n *\n * @param storageUri - file:// URI (e.g. \"file://docs/overview.md\")\n * @param expectedChecksum - Optional SHA-256 to verify against\n * @returns Stored resource metadata\n * @throws ChecksumMismatchError if expectedChecksum is provided and does not match\n * @throws Error if file does not exist\n */\n async register(storageUri: string, expectedChecksum?: string, options?: { noGit?: boolean }): Promise<StoredResource> {\n const filePath = this.resolveUri(storageUri);\n\n this.logger?.debug('Registering resource', { storageUri });\n\n const content = await fs.readFile(filePath);\n const checksum = calculateChecksum(content);\n\n if (expectedChecksum !== undefined && !verifyChecksum(content, expectedChecksum)) {\n throw new ChecksumMismatchError(storageUri, expectedChecksum, checksum);\n }\n\n if (this.shouldRunGit(options?.noGit)) {\n execFileSync('git', ['add', filePath], { cwd: this.projectRoot });\n }\n\n this.logger?.info('Resource registered', { storageUri, checksum, byteSize: content.length });\n\n return {\n storageUri,\n checksum,\n byteSize: content.length,\n created: new Date().toISOString(),\n };\n }\n\n /**\n * Read file content by URI.\n *\n * @param storageUri - file:// URI\n * @returns Raw bytes\n */\n async retrieve(storageUri: string): Promise<Buffer> {\n const filePath = this.resolveUri(storageUri);\n try {\n return await fs.readFile(filePath);\n } catch (error: any) {\n if (error.code === 'ENOENT') {\n throw new Error(`Resource not found: ${storageUri}`);\n }\n throw error;\n }\n }\n\n /**\n * Move a file from one URI to another.\n *\n * If .git/ exists in the project root and noGit is not set, runs `git mv`.\n * Otherwise (no .git/ or noGit: true), runs fs.rename.\n *\n * @param fromUri - Current file:// URI\n * @param toUri - New file:// URI\n * @param options.noGit - Skip git mv even if .git/ is present\n */\n async move(fromUri: string, toUri: string, options?: { noGit?: boolean }): Promise<void> {\n const fromPath = this.resolveUri(fromUri);\n const toPath = this.resolveUri(toUri);\n\n this.logger?.debug('Moving resource', { fromUri, toUri });\n\n await fs.mkdir(path.dirname(toPath), { recursive: true });\n\n if (this.shouldRunGit(options?.noGit)) {\n // git mv handles both the filesystem rename and the index update\n execFileSync('git', ['mv', fromPath, toPath], { cwd: this.projectRoot });\n } else {\n await fs.rename(fromPath, toPath);\n }\n\n this.logger?.info('Resource moved', { fromUri, toUri });\n }\n\n /**\n * Remove a file from the working tree.\n *\n * If .git/ exists and noGit is not set:\n * - keepFile false (default): runs `git rm` (removes from index and disk)\n * - keepFile true: runs `git rm --cached` (removes from index only, file stays on disk)\n * If no .git/ or noGit: true:\n * - keepFile false: runs fs.unlink\n * - keepFile true: no-op on filesystem\n *\n * @param storageUri - file:// URI\n * @param options.noGit - Skip git rm even if .git/ is present\n * @param options.keepFile - Remove from git index only; leave file on disk\n */\n async remove(storageUri: string, options?: { noGit?: boolean; keepFile?: boolean }): Promise<void> {\n const filePath = this.resolveUri(storageUri);\n const keepFile = options?.keepFile ?? false;\n\n this.logger?.debug('Removing resource', { storageUri, keepFile });\n\n const useGit = this.shouldRunGit(options?.noGit);\n\n if (useGit) {\n const gitArgs = keepFile\n ? ['rm', '--cached', filePath]\n : ['rm', filePath];\n execFileSync('git', gitArgs, { cwd: this.projectRoot });\n this.logger?.info('Resource removed', { storageUri, keepFile, git: true });\n return;\n }\n\n if (keepFile) {\n this.logger?.info('Resource removed from index (file kept on disk)', { storageUri });\n return;\n }\n\n try {\n await fs.unlink(filePath);\n this.logger?.info('Resource removed', { storageUri });\n } catch (error: any) {\n if (error.code === 'ENOENT') {\n this.logger?.warn('Resource file already absent', { storageUri });\n return;\n }\n throw error;\n }\n }\n\n /**\n * Convert a file:// URI to an absolute filesystem path.\n *\n * \"file://docs/overview.md\" → \"{projectRoot}/docs/overview.md\"\n *\n * @param storageUri - file:// URI\n * @returns Absolute path\n */\n resolveUri(storageUri: string): string {\n if (!storageUri.startsWith('file://')) {\n throw new Error(`Invalid storage URI (must start with file://): ${storageUri}`);\n }\n const relativePath = storageUri.slice('file://'.length);\n return path.join(this.projectRoot, relativePath);\n }\n}\n\n/**\n * Thrown when a registered file's checksum does not match the expected value.\n * This indicates the file on disk differs from what was recorded (e.g. modified\n * after staging, or wrong file path provided).\n */\nexport class ChecksumMismatchError extends Error {\n constructor(\n readonly storageUri: string,\n readonly expected: string,\n readonly actual: string,\n ) {\n super(\n `Checksum mismatch for ${storageUri}: expected ${expected.slice(0, 8)}... but got ${actual.slice(0, 8)}...\\n` +\n `The file on disk differs from the recorded checksum. Has it been modified since staging?`\n );\n this.name = 'ChecksumMismatchError';\n }\n}\n","/**\n * Checksum utilities for content verification\n */\n\nimport { createHash } from 'crypto';\n\n/**\n * Calculate SHA-256 checksum of content\n * @param content The content to hash\n * @returns Hex-encoded SHA-256 hash\n */\nexport function calculateChecksum(content: string | Buffer): string {\n const hash = createHash('sha256');\n hash.update(content);\n return hash.digest('hex');\n}\n\n/**\n * Verify content against a checksum\n * @param content The content to verify\n * @param checksum The expected checksum\n * @returns True if content matches checksum\n */\nexport function verifyChecksum(content: string | Buffer, checksum: string): boolean {\n return calculateChecksum(content) === checksum;\n}\n","/**\n * Storage URI Derivation\n *\n * Builds the file:// URI a resource lives at in the working tree from its\n * name and validated media type. Extensions come from the media-type\n * registry in @semiont/core; formats are validated upstream at the\n * create/yield boundary, so the lookup is strict — no fallback.\n */\n\nimport { MEDIA_TYPES, type SupportedMediaType } from '@semiont/core';\n\n/**\n * Derive a file:// storage URI from a resource name and media type.\n *\n * The name is lowercased, runs of non-alphanumeric characters collapse to\n * single hyphens, and leading/trailing hyphens are stripped.\n *\n * @example\n * deriveStorageUri(\"My Document\", \"text/markdown\") // => \"file://my-document.md\"\n */\nexport function deriveStorageUri(name: string, format: SupportedMediaType): string {\n const slug = name\n .toLowerCase()\n .replace(/[^a-z0-9]+/g, '-')\n .replace(/^-|-$/g, '');\n return `file://${slug}${MEDIA_TYPES[format].extension}`;\n}\n","/**\n * PDF Text Layer Extraction\n *\n * Extracts positioned text from native, non-scanned PDFs using pdfjs-dist.\n * Returns null for scanned/image-only PDFs (no text items).\n *\n * Coordinates are in PDF point space, originating from the bottom-left.\n * The Y-flip to canvas pixels happens downstream.\n */\n\nimport * as pdfjs from 'pdfjs-dist/legacy/build/pdf.mjs';\nimport type { PdfTextLayer, PdfPageInfo, PdfTextItem } from './pdf-text-layer';\n\nexport async function extractPdfTextLayer(\n bytes: Uint8Array | Buffer\n): Promise<PdfTextLayer | null> {\n // pdf.js v5 removed the isEvalSupported option; this path only calls\n // getTextContent (no rendering / no PDF functions).\n const doc = await pdfjs.getDocument({ data: bytes }).promise;\n\n try {\n const pages: PdfPageInfo[] = [];\n const items: PdfTextItem[] = [];\n let text = '';\n let hasAnyTextItems = false;\n\n for (let pageNum = 1; pageNum <= doc.numPages; pageNum++) {\n const page = await doc.getPage(pageNum);\n const viewport = page.getViewport({ scale: 1.0 });\n const content = await page.getTextContent(); // all text items on the page\n\n pages.push({\n pageNumber: pageNum,\n widthPt: viewport.width,\n heightPt: viewport.height,\n });\n\n for (const item of content.items) {\n if (!('str' in item)) continue; // skip marked-content items (no text)\n\n if (item.str.trim()) {\n hasAnyTextItems = true;\n const start = text.length;\n text += item.str;\n const end = text.length; // range covers only this run's own chars\n\n const [, , , , x, y] = item.transform as number[];\n\n items.push({\n start,\n end,\n page: pageNum,\n x,\n y,\n width: item.width,\n height: item.height,\n });\n\n // Separator AFTER recording the run, so its [start, end) never\n // includes it. pdf.js flags the last run on a line with hasEOL —\n // newline there, space between words otherwise, so reading-order\n // lines don't glue (e.g. \"textsecond\").\n text += item.hasEOL ? '\\n' : ' ';\n } else if (item.hasEOL) {\n // Standalone end-of-line marker (empty/whitespace str): keep the\n // line break without letting whitespace-only runs add stray spaces.\n text += '\\n';\n }\n }\n\n text += '\\n'; // page break\n }\n\n if (!hasAnyTextItems) return null;\n\n return { pages, text, items };\n } finally {\n // Release the pdf.js document — Phase 2 runs this in a long-lived worker pool.\n await doc.destroy();\n }\n}\n","import type { PdfCoordinate } from '@semiont/core';\nimport type { PdfTextLayer, PdfTextItem } from './pdf-text-layer';\n\n/**\n * Items whose baseline Y is within this many PDF points are treated as being on\n * the same line. Tuned for ~12pt body text; revisit for documents with large or\n * variable font sizes (Phase 4 / #738).\n */\nconst SAME_LINE_THRESHOLD_PT = 2;\n\n/**\n * Locates bounding rectangles for a span of text in a PdfTextLayer\n * (single-line or multi-line).\n * \n * Finds all overlapping items [start, end), groups them by page and line,\n * and records one bounding rectangle per line as a PdfCoordinate.\n * \n * Returns array of PdfCoordinate, one per line of text covered by the span.\n * Returns empty array if no items overlap the span.\n */\nexport function locate(\n layer: PdfTextLayer,\n start: number,\n end: number\n): PdfCoordinate[] {\n const overlap: PdfTextItem[] = layer.items.filter(\n item => item.start < end && item.end > start\n );\n if (overlap.length === 0) return [];\n\n const pages: Map<number, PdfTextItem[]> = groupItemsByPage(overlap);\n const rects: PdfCoordinate[] = [];\n\n // for each page, group items into lines and compute one rectangle per line\n for (const [page, pageItems] of pages) {\n const lines = groupItemsByLine(pageItems, SAME_LINE_THRESHOLD_PT);\n // Compute one bounding rectangle per line and add it to rects\n for (const lineItems of lines) {\n const x = Math.min(...lineItems.map(i => i.x));\n const right = Math.max(...lineItems.map(i => i.x + i.width));\n const y = Math.min(...lineItems.map(i => i.y));\n const top = Math.max(...lineItems.map(i => i.y + i.height));\n rects.push({page, x, y, width: right - x, height: top - y});\n }\n }\n return rects;\n}\n\nfunction groupItemsByPage(items: PdfTextItem[]): Map<number, PdfTextItem[]> {\n const map = new Map<number, PdfTextItem[]>();\n for (const item of items) {\n const existing = map.get(item.page);\n if (existing) {\n existing.push(item);\n } else {\n map.set(item.page, [item]);\n }\n }\n return map;\n}\n\n\n/**\n * Sorts text items into lines when their y coordinates are\n * within `sameLineThreshold` points of each other.\n * Sorted top-to-bottom (descending y in PDF space), then left-to-right.\n * \n * Returns 2D array: \n * Outer array = list of lines\n * Inner array = list of items on that line\n*/\nfunction groupItemsByLine(items: PdfTextItem[], sameLineThreshold: number): PdfTextItem[][] {\n // Sort top-to-bottom by y; if y is equal, sort left-to-right by x\n const sorted = [...items].sort((a, b) => b.y - a.y || a.x - b.x);\n const lines: PdfTextItem[][] = [];\n let currentLine: PdfTextItem[] = [];\n\n for (const item of sorted) {\n if (currentLine.length === 0 || Math.abs(item.y - currentLine[0].y) <= sameLineThreshold) {\n currentLine.push(item);\n } else {\n lines.push(currentLine);\n currentLine = [item];\n }\n }\n if (currentLine.length > 0) lines.push(currentLine);\n return lines;\n}\n"],"mappings":";AAsBA,SAAS,YAAY,UAAU;AAC/B,SAAS,oBAAoB;AAC7B,OAAO,UAAU;;;ACpBjB,SAAS,kBAAkB;AAOpB,SAAS,kBAAkB,SAAkC;AAClE,QAAM,OAAO,WAAW,QAAQ;AAChC,OAAK,OAAO,OAAO;AACnB,SAAO,KAAK,OAAO,KAAK;AAC1B;AAQO,SAAS,eAAe,SAA0B,UAA2B;AAClF,SAAO,kBAAkB,OAAO,MAAM;AACxC;;;ADiBO,IAAM,mBAAN,MAAuB;AAAA,EACpB;AAAA,EACA;AAAA,EACA;AAAA,EAER,YAAY,SAAyB,QAAiB;AACpD,SAAK,cAAc,QAAQ;AAC3B,SAAK,UAAU,QAAQ;AACvB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEQ,aAAa,OAA0B;AAC7C,WAAO,KAAK,WAAW,CAAC;AAAA,EAC1B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWA,MAAM,MAAM,SAAiB,YAAoB,SAAwD;AACvG,UAAM,WAAW,KAAK,WAAW,UAAU;AAC3C,UAAM,WAAW,kBAAkB,OAAO;AAE1C,SAAK,QAAQ,MAAM,oBAAoB,EAAE,YAAY,UAAU,QAAQ,OAAO,CAAC;AAE/E,UAAM,GAAG,MAAM,KAAK,QAAQ,QAAQ,GAAG,EAAE,WAAW,KAAK,CAAC;AAC1D,UAAM,GAAG,UAAU,UAAU,OAAO;AAEpC,QAAI,KAAK,aAAa,SAAS,KAAK,GAAG;AACrC,mBAAa,OAAO,CAAC,OAAO,QAAQ,GAAG,EAAE,KAAK,KAAK,YAAY,CAAC;AAAA,IAClE;AAEA,SAAK,QAAQ,KAAK,mBAAmB,EAAE,YAAY,UAAU,UAAU,QAAQ,OAAO,CAAC;AAEvF,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA,UAAU,QAAQ;AAAA,MAClB,UAAS,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAcA,MAAM,SAAS,YAAoB,kBAA2B,SAAwD;AACpH,UAAM,WAAW,KAAK,WAAW,UAAU;AAE3C,SAAK,QAAQ,MAAM,wBAAwB,EAAE,WAAW,CAAC;AAEzD,UAAM,UAAU,MAAM,GAAG,SAAS,QAAQ;AAC1C,UAAM,WAAW,kBAAkB,OAAO;AAE1C,QAAI,qBAAqB,UAAa,CAAC,eAAe,SAAS,gBAAgB,GAAG;AAChF,YAAM,IAAI,sBAAsB,YAAY,kBAAkB,QAAQ;AAAA,IACxE;AAEA,QAAI,KAAK,aAAa,SAAS,KAAK,GAAG;AACrC,mBAAa,OAAO,CAAC,OAAO,QAAQ,GAAG,EAAE,KAAK,KAAK,YAAY,CAAC;AAAA,IAClE;AAEA,SAAK,QAAQ,KAAK,uBAAuB,EAAE,YAAY,UAAU,UAAU,QAAQ,OAAO,CAAC;AAE3F,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA,UAAU,QAAQ;AAAA,MAClB,UAAS,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAM,SAAS,YAAqC;AAClD,UAAM,WAAW,KAAK,WAAW,UAAU;AAC3C,QAAI;AACF,aAAO,MAAM,GAAG,SAAS,QAAQ;AAAA,IACnC,SAAS,OAAY;AACnB,UAAI,MAAM,SAAS,UAAU;AAC3B,cAAM,IAAI,MAAM,uBAAuB,UAAU,EAAE;AAAA,MACrD;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,MAAM,KAAK,SAAiB,OAAe,SAA8C;AACvF,UAAM,WAAW,KAAK,WAAW,OAAO;AACxC,UAAM,SAAS,KAAK,WAAW,KAAK;AAEpC,SAAK,QAAQ,MAAM,mBAAmB,EAAE,SAAS,MAAM,CAAC;AAExD,UAAM,GAAG,MAAM,KAAK,QAAQ,MAAM,GAAG,EAAE,WAAW,KAAK,CAAC;AAExD,QAAI,KAAK,aAAa,SAAS,KAAK,GAAG;AAErC,mBAAa,OAAO,CAAC,MAAM,UAAU,MAAM,GAAG,EAAE,KAAK,KAAK,YAAY,CAAC;AAAA,IACzE,OAAO;AACL,YAAM,GAAG,OAAO,UAAU,MAAM;AAAA,IAClC;AAEA,SAAK,QAAQ,KAAK,kBAAkB,EAAE,SAAS,MAAM,CAAC;AAAA,EACxD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAgBA,MAAM,OAAO,YAAoB,SAAkE;AACjG,UAAM,WAAW,KAAK,WAAW,UAAU;AAC3C,UAAM,WAAW,SAAS,YAAY;AAEtC,SAAK,QAAQ,MAAM,qBAAqB,EAAE,YAAY,SAAS,CAAC;AAEhE,UAAM,SAAS,KAAK,aAAa,SAAS,KAAK;AAE/C,QAAI,QAAQ;AACV,YAAM,UAAU,WACZ,CAAC,MAAM,YAAY,QAAQ,IAC3B,CAAC,MAAM,QAAQ;AACnB,mBAAa,OAAO,SAAS,EAAE,KAAK,KAAK,YAAY,CAAC;AACtD,WAAK,QAAQ,KAAK,oBAAoB,EAAE,YAAY,UAAU,KAAK,KAAK,CAAC;AACzE;AAAA,IACF;AAEA,QAAI,UAAU;AACZ,WAAK,QAAQ,KAAK,mDAAmD,EAAE,WAAW,CAAC;AACnF;AAAA,IACF;AAEA,QAAI;AACF,YAAM,GAAG,OAAO,QAAQ;AACxB,WAAK,QAAQ,KAAK,oBAAoB,EAAE,WAAW,CAAC;AAAA,IACtD,SAAS,OAAY;AACnB,UAAI,MAAM,SAAS,UAAU;AAC3B,aAAK,QAAQ,KAAK,gCAAgC,EAAE,WAAW,CAAC;AAChE;AAAA,MACF;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,WAAW,YAA4B;AACrC,QAAI,CAAC,WAAW,WAAW,SAAS,GAAG;AACrC,YAAM,IAAI,MAAM,kDAAkD,UAAU,EAAE;AAAA,IAChF;AACA,UAAM,eAAe,WAAW,MAAM,UAAU,MAAM;AACtD,WAAO,KAAK,KAAK,KAAK,aAAa,YAAY;AAAA,EACjD;AACF;AAOO,IAAM,wBAAN,cAAoC,MAAM;AAAA,EAC/C,YACW,YACA,UACA,QACT;AACA;AAAA,MACE,yBAAyB,UAAU,cAAc,SAAS,MAAM,GAAG,CAAC,CAAC,eAAe,OAAO,MAAM,GAAG,CAAC,CAAC;AAAA;AAAA,IAExG;AAPS;AACA;AACA;AAMT,SAAK,OAAO;AAAA,EACd;AAAA,EATW;AAAA,EACA;AAAA,EACA;AAQb;;;AEtPA,SAAS,mBAA4C;AAW9C,SAAS,iBAAiB,MAAc,QAAoC;AACjF,QAAM,OAAO,KACV,YAAY,EACZ,QAAQ,eAAe,GAAG,EAC1B,QAAQ,UAAU,EAAE;AACvB,SAAO,UAAU,IAAI,GAAG,YAAY,MAAM,EAAE,SAAS;AACvD;;;AChBA,YAAY,WAAW;AAGvB,eAAsB,oBAClB,OAC4B;AAG5B,QAAM,MAAM,MAAY,kBAAY,EAAE,MAAM,MAAM,CAAC,EAAE;AAErD,MAAI;AACA,UAAM,QAAuB,CAAC;AAC9B,UAAM,QAAuB,CAAC;AAC9B,QAAI,OAAO;AACX,QAAI,kBAAkB;AAEtB,aAAS,UAAU,GAAG,WAAW,IAAI,UAAU,WAAW;AACtD,YAAM,OAAO,MAAM,IAAI,QAAQ,OAAO;AACtC,YAAM,WAAW,KAAK,YAAY,EAAE,OAAO,EAAI,CAAC;AAChD,YAAM,UAAU,MAAM,KAAK,eAAe;AAE1C,YAAM,KAAK;AAAA,QACP,YAAY;AAAA,QACZ,SAAS,SAAS;AAAA,QAClB,UAAU,SAAS;AAAA,MACvB,CAAC;AAED,iBAAW,QAAQ,QAAQ,OAAO;AAC9B,YAAI,EAAE,SAAS,MAAO;AAEtB,YAAI,KAAK,IAAI,KAAK,GAAG;AACjB,4BAAkB;AAClB,gBAAM,QAAQ,KAAK;AACnB,kBAAQ,KAAK;AACb,gBAAM,MAAM,KAAK;AAEjB,gBAAM,CAAC,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,IAAI,KAAK;AAE5B,gBAAM,KAAK;AAAA,YACP;AAAA,YACA;AAAA,YACA,MAAM;AAAA,YACN;AAAA,YACA;AAAA,YACA,OAAO,KAAK;AAAA,YACZ,QAAQ,KAAK;AAAA,UACjB,CAAC;AAMD,kBAAQ,KAAK,SAAS,OAAO;AAAA,QACjC,WAAW,KAAK,QAAQ;AAGpB,kBAAQ;AAAA,QACZ;AAAA,MACJ;AAEA,cAAQ;AAAA,IACZ;AAEA,QAAI,CAAC,gBAAiB,QAAO;AAE7B,WAAO,EAAE,OAAO,MAAM,MAAM;AAAA,EAChC,UAAE;AAEE,UAAM,IAAI,QAAQ;AAAA,EACtB;AACJ;;;ACxEA,IAAM,yBAAyB;AAYxB,SAAS,OACZ,OACA,OACA,KACe;AACf,QAAM,UAAyB,MAAM,MAAM;AAAA,IACvC,UAAQ,KAAK,QAAQ,OAAO,KAAK,MAAM;AAAA,EAC3C;AACA,MAAI,QAAQ,WAAW,EAAG,QAAO,CAAC;AAElC,QAAM,QAAoC,iBAAiB,OAAO;AAClE,QAAM,QAAyB,CAAC;AAGhC,aAAW,CAAC,MAAM,SAAS,KAAK,OAAO;AACnC,UAAM,QAAQ,iBAAiB,WAAW,sBAAsB;AAEhE,eAAW,aAAa,OAAO;AAC3B,YAAM,IAAI,KAAK,IAAI,GAAG,UAAU,IAAI,OAAK,EAAE,CAAC,CAAC;AAC7C,YAAM,QAAQ,KAAK,IAAI,GAAG,UAAU,IAAI,OAAK,EAAE,IAAI,EAAE,KAAK,CAAC;AAC3D,YAAM,IAAI,KAAK,IAAI,GAAG,UAAU,IAAI,OAAK,EAAE,CAAC,CAAC;AAC7C,YAAM,MAAM,KAAK,IAAI,GAAG,UAAU,IAAI,OAAK,EAAE,IAAI,EAAE,MAAM,CAAC;AAC1D,YAAM,KAAK,EAAC,MAAM,GAAG,GAAG,OAAO,QAAQ,GAAG,QAAQ,MAAM,EAAC,CAAC;AAAA,IAC9D;AAAA,EACJ;AACA,SAAO;AACX;AAEA,SAAS,iBAAiB,OAAkD;AACxE,QAAM,MAAM,oBAAI,IAA2B;AAC3C,aAAW,QAAQ,OAAO;AACtB,UAAM,WAAW,IAAI,IAAI,KAAK,IAAI;AAClC,QAAI,UAAU;AACV,eAAS,KAAK,IAAI;AAAA,IACtB,OAAO;AACH,UAAI,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC;AAAA,IAC7B;AAAA,EACJ;AACA,SAAO;AACX;AAYA,SAAS,iBAAiB,OAAsB,mBAA4C;AAExF,QAAM,SAAS,CAAC,GAAG,KAAK,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AAC/D,QAAM,QAAyB,CAAC;AAChC,MAAI,cAA6B,CAAC;AAElC,aAAW,QAAQ,QAAQ;AACvB,QAAI,YAAY,WAAW,KAAK,KAAK,IAAI,KAAK,IAAI,YAAY,CAAC,EAAE,CAAC,KAAK,mBAAmB;AACtF,kBAAY,KAAK,IAAI;AAAA,IACzB,OAAO;AACH,YAAM,KAAK,WAAW;AACtB,oBAAc,CAAC,IAAI;AAAA,IACvB;AAAA,EACJ;AACA,MAAI,YAAY,SAAS,EAAG,OAAM,KAAK,WAAW;AAClD,SAAO;AACX;","names":[]}
|
package/package.json
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@semiont/content",
|
|
3
|
-
"version": "0.5.
|
|
3
|
+
"version": "0.5.7",
|
|
4
4
|
"engines": {
|
|
5
5
|
"node": ">=24.0.0"
|
|
6
6
|
},
|
|
7
7
|
"type": "module",
|
|
8
|
-
"description": "
|
|
8
|
+
"description": "Working-tree storage for project resources and PDF text-layer extraction",
|
|
9
9
|
"main": "./dist/index.js",
|
|
10
10
|
"types": "./dist/index.d.ts",
|
|
11
11
|
"exports": {
|
|
@@ -42,9 +42,9 @@
|
|
|
42
42
|
"keywords": [
|
|
43
43
|
"content",
|
|
44
44
|
"storage",
|
|
45
|
-
"
|
|
46
|
-
"
|
|
47
|
-
"
|
|
45
|
+
"working-tree",
|
|
46
|
+
"checksum",
|
|
47
|
+
"pdf",
|
|
48
48
|
"semiont"
|
|
49
49
|
],
|
|
50
50
|
"author": "The AI Alliance",
|