omni-viewer-core 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/i18n/catalog.en.js +13 -0
- package/dist/i18n/catalog.ja.js +13 -1
- package/dist/i18n/catalog.ko.js +13 -0
- package/dist/i18n/catalog.zh-cn.js +10 -0
- package/dist/parsers/safetensors/index.d.ts +34 -0
- package/dist/parsers/safetensors/index.js +233 -0
- package/dist/parsers/toml/model.d.ts +10 -0
- package/dist/parsers/toml/parse.js +104 -34
- package/dist/registry/index.d.ts +1 -0
- package/dist/registry/index.js +5 -1
- package/dist/styles/pdf.css +177 -0
- package/dist/styles/safetensors.css +3 -0
- package/dist/styles/toml.css +1 -1
- package/dist/styles/yaml.css +1 -1
- package/dist/viewers/pdf/controller.d.ts +8 -0
- package/dist/viewers/pdf/controller.js +4 -0
- package/dist/viewers/pdf/editing.d.ts +11 -0
- package/dist/viewers/pdf/editing.js +19 -16
- package/dist/viewers/pdf/index.d.ts +3 -2
- package/dist/viewers/pdf/index.js +195 -23
- package/dist/viewers/pdf/styles.d.ts +1 -1
- package/dist/viewers/pdf/styles.js +177 -0
- package/dist/viewers/safetensors/index.d.ts +19 -0
- package/dist/viewers/safetensors/index.js +170 -0
- package/dist/viewers/safetensors/styles.d.ts +1 -0
- package/dist/viewers/safetensors/styles.js +4 -0
- package/dist/viewers/structured-styles.d.ts +1 -1
- package/dist/viewers/structured-styles.js +1 -1
- package/dist/viewers/structured.d.ts +15 -0
- package/dist/viewers/structured.js +196 -47
- package/dist/viewers/toml/controller.d.ts +27 -1
- package/dist/viewers/toml/controller.js +64 -10
- package/package.json +9 -1
package/README.md
CHANGED
|
@@ -15,7 +15,7 @@ models, viewers mount into a DOM element, and everything host-specific
|
|
|
15
15
|
- **Documents** — PDF, Word (DOCX and legacy DOC), HWP, PowerPoint (PPTX and
|
|
16
16
|
legacy PPT), Markdown
|
|
17
17
|
- **Data & spreadsheets** — Excel, CSV/TSV, JSON, JSONL/NDJSON, YAML, TOML,
|
|
18
|
-
Parquet, Avro, HDF5, MATLAB MAT, Protocol Buffers, ReqIF, SQLite
|
|
18
|
+
Parquet, Avro, HDF5, MATLAB MAT, Safetensors, Protocol Buffers, ReqIF, SQLite
|
|
19
19
|
- **Media & graphics** — audio (waveform/spectrogram), video, images,
|
|
20
20
|
Photoshop PSD
|
|
21
21
|
- **Engineering & automotive** — CAN DBC, AUTOSAR ARXML, ASAM A2L, Vector
|
package/dist/i18n/catalog.en.js
CHANGED
|
@@ -50,6 +50,10 @@ export const CATALOG_EN = {
|
|
|
50
50
|
'hdf5.structure': 'Structure', 'hdf5.noData': 'No data available.',
|
|
51
51
|
'hdf5.matchingRows': '{count} matching rows',
|
|
52
52
|
'hdf5.matchingRowsLimited': '{visible} / {total} matching rows shown',
|
|
53
|
+
'safetensors.title': 'Safetensors', 'safetensors.search': 'Search tensors…', 'safetensors.copyJson': 'Copy JSON',
|
|
54
|
+
'safetensors.structure': 'Structure', 'safetensors.noData': 'No data available.',
|
|
55
|
+
'safetensors.matchingRows': '{count} matching rows',
|
|
56
|
+
'safetensors.matchingRowsLimited': '{visible} / {total} matching rows shown',
|
|
53
57
|
'proto.title': 'Protocol Buffer', 'proto.summary': '{syntax} · {package} · {messages} messages · {services} services',
|
|
54
58
|
'proto.noPackage': 'no package', 'proto.search': 'Search message, field, enum, RPC', 'proto.copyPanel': 'Copy panel',
|
|
55
59
|
'proto.panel.tree': 'Message Tree', 'proto.panel.types': 'Types', 'proto.panel.relationships': 'Relationships',
|
|
@@ -149,6 +153,9 @@ export const CATALOG_EN = {
|
|
|
149
153
|
'structured.copyValue': 'Value', 'structured.copyJson': 'JSON',
|
|
150
154
|
'structured.source': 'Source', 'structured.sourceDescription': 'Editable original text',
|
|
151
155
|
'structured.treeDescription': 'Structured preview',
|
|
156
|
+
'structured.scopeLabel': 'Search in', 'structured.scope.all': 'All', 'structured.scope.key': 'Key',
|
|
157
|
+
'structured.scope.path': 'Path', 'structured.scope.value': 'Value',
|
|
158
|
+
'structured.matchCount': '{count} matches', 'structured.keyCount': '{count} keys', 'structured.itemCount': '{count} items',
|
|
152
159
|
'toml.treeView': 'TOML', 'yaml.treeView': 'YAML', 'jsonl.title': 'JSON Lines',
|
|
153
160
|
'jsonl.loadMore': 'Load more', 'jsonl.loadAll': 'Load all', 'jsonl.save': 'Save',
|
|
154
161
|
'jsonl.editUnavailable': 'Editing requires the complete file.', 'diag.jsonl.load-failed': 'Could not load more JSON Lines data.',
|
|
@@ -328,6 +335,7 @@ export const CATALOG_EN = {
|
|
|
328
335
|
'pdf.goToPage': 'Go to page',
|
|
329
336
|
'pdf.zoomIn': 'Zoom in',
|
|
330
337
|
'pdf.zoomOut': 'Zoom out',
|
|
338
|
+
'pdf.thumbnails': 'Page thumbnails',
|
|
331
339
|
'pdf.viewOptions': 'View options',
|
|
332
340
|
'pdf.fitWidth': 'Fit to width',
|
|
333
341
|
'pdf.fitHeight': 'Fit to height',
|
|
@@ -369,6 +377,11 @@ export const CATALOG_EN = {
|
|
|
369
377
|
'pdf.textPlace': 'Click a page to place the text.',
|
|
370
378
|
'pdf.deletePage': 'Delete page',
|
|
371
379
|
'pdf.deleteAnnotation': 'Delete annotation',
|
|
380
|
+
'pdf.markupList': 'Annotations',
|
|
381
|
+
'pdf.closeMarkupList': 'Close annotations',
|
|
382
|
+
'pdf.changeColor': 'Change color',
|
|
383
|
+
'pdf.copyText': 'Copy text',
|
|
384
|
+
'pdf.noMarkups': 'No highlights, underlines, or strikethroughs yet.',
|
|
372
385
|
'pdf.password': 'Password',
|
|
373
386
|
'pdf.passwordRequired': 'This PDF requires a password.',
|
|
374
387
|
'pdf.passwordIncorrect': 'Incorrect password. Try again.',
|
package/dist/i18n/catalog.ja.js
CHANGED
|
@@ -46,6 +46,10 @@ export const CATALOG_JA = {
|
|
|
46
46
|
'hdf5.structure': '構造', 'hdf5.noData': '表示できるデータがありません。',
|
|
47
47
|
'hdf5.matchingRows': '一致する行: {count}件',
|
|
48
48
|
'hdf5.matchingRowsLimited': '一致する{total}行中{visible}行を表示',
|
|
49
|
+
'safetensors.title': 'Safetensors', 'safetensors.search': 'テンソルを検索…', 'safetensors.copyJson': 'JSONをコピー',
|
|
50
|
+
'safetensors.structure': '構造', 'safetensors.noData': '表示できるデータがありません。',
|
|
51
|
+
'safetensors.matchingRows': '一致する行: {count}件',
|
|
52
|
+
'safetensors.matchingRowsLimited': '一致する{total}行中{visible}行を表示',
|
|
49
53
|
'proto.title': 'Protocol Buffer', 'proto.summary': '{syntax} · {package} · メッセージ{messages}件 · サービス{services}件',
|
|
50
54
|
'proto.noPackage': 'パッケージなし', 'proto.search': 'メッセージ、フィールド、列挙型、RPCを検索', 'proto.copyPanel': 'パネルをコピー',
|
|
51
55
|
'proto.panel.tree': 'メッセージツリー', 'proto.panel.types': '型', 'proto.panel.relationships': '関係',
|
|
@@ -153,6 +157,9 @@ export const CATALOG_JA = {
|
|
|
153
157
|
'structured.copyValue': '値', 'structured.copyJson': 'JSON',
|
|
154
158
|
'structured.source': 'ソース', 'structured.sourceDescription': '編集可能な元のテキスト',
|
|
155
159
|
'structured.treeDescription': '構造化プレビュー',
|
|
160
|
+
'structured.scopeLabel': '検索対象', 'structured.scope.all': 'すべて', 'structured.scope.key': 'キー',
|
|
161
|
+
'structured.scope.path': 'パス', 'structured.scope.value': '値',
|
|
162
|
+
'structured.matchCount': '{count} 件一致', 'structured.keyCount': '{count} 個のキー', 'structured.itemCount': '{count} 個の項目',
|
|
156
163
|
'toml.treeView': 'TOML', 'yaml.treeView': 'YAML', 'jsonl.title': 'JSON Lines',
|
|
157
164
|
'jsonl.loadMore': 'さらに読み込む', 'jsonl.loadAll': 'すべて読み込む', 'jsonl.save': '保存',
|
|
158
165
|
'jsonl.editUnavailable': '編集には完全なファイルが必要です。', 'diag.jsonl.load-failed': 'JSON Linesデータを追加で読み込めませんでした。',
|
|
@@ -243,6 +250,7 @@ export const CATALOG_JA = {
|
|
|
243
250
|
'json.result.copy': 'コピー', 'json.result.replace': 'エディターを置換', 'json.result.close': '閉じる',
|
|
244
251
|
'pdf.title': 'PDF', 'pdf.loading': 'PDFを読み込み中…', 'pdf.loadFailed': 'このPDFを読み込めません。',
|
|
245
252
|
'pdf.pages': 'ページ', 'pdf.goToPage': 'ページへ移動', 'pdf.zoomIn': '拡大', 'pdf.zoomOut': '縮小',
|
|
253
|
+
'pdf.thumbnails': 'ページサムネイル',
|
|
246
254
|
'pdf.viewOptions': '表示オプション', 'pdf.fitWidth': '幅に合わせる', 'pdf.fitHeight': '高さに合わせる',
|
|
247
255
|
'pdf.singlePage': '単一ページ', 'pdf.twoPagesOdd': '見開き(奇数ページを左)',
|
|
248
256
|
'pdf.twoPagesEven': '見開き(偶数ページを左)', 'pdf.matchTheme': 'テーマの色に合わせる',
|
|
@@ -265,7 +273,11 @@ export const CATALOG_JA = {
|
|
|
265
273
|
'pdf.markupOptions': 'テキストマークアップのオプション', 'pdf.underline': '下線',
|
|
266
274
|
'pdf.underlineColor': '下線の色', 'pdf.strikeout': '取り消し線', 'pdf.strikeoutColor': '取り消し線の色',
|
|
267
275
|
'pdf.cancel': 'キャンセル', 'pdf.textPlace': 'ページをクリックしてテキストを配置してください。',
|
|
268
|
-
'pdf.deletePage': 'ページを削除', 'pdf.deleteAnnotation': '注釈を削除',
|
|
276
|
+
'pdf.deletePage': 'ページを削除', 'pdf.deleteAnnotation': '注釈を削除',
|
|
277
|
+
'pdf.markupList': '注釈一覧', 'pdf.closeMarkupList': '注釈一覧を閉じる',
|
|
278
|
+
'pdf.changeColor': '色を変更', 'pdf.copyText': 'テキストをコピー',
|
|
279
|
+
'pdf.noMarkups': 'ハイライト・下線・取り消し線はまだありません。',
|
|
280
|
+
'pdf.password': 'パスワード',
|
|
269
281
|
'pdf.passwordRequired': 'このPDFにはパスワードが必要です。',
|
|
270
282
|
'pdf.passwordIncorrect': 'パスワードが正しくありません。もう一度お試しください。',
|
|
271
283
|
'pdf.passwordSubmit': '開く', 'pdf.editingUnavailable': 'この環境ではPDFを編集できません',
|
package/dist/i18n/catalog.ko.js
CHANGED
|
@@ -48,6 +48,10 @@ export const CATALOG_KO = {
|
|
|
48
48
|
'hdf5.structure': '구조', 'hdf5.noData': '표시할 데이터가 없습니다.',
|
|
49
49
|
'hdf5.matchingRows': '일치하는 행 {count}개',
|
|
50
50
|
'hdf5.matchingRowsLimited': '전체 {total}행 중 일치하는 {visible}행 표시',
|
|
51
|
+
'safetensors.title': 'Safetensors', 'safetensors.search': '텐서 검색…', 'safetensors.copyJson': 'JSON 복사',
|
|
52
|
+
'safetensors.structure': '구조', 'safetensors.noData': '표시할 데이터가 없습니다.',
|
|
53
|
+
'safetensors.matchingRows': '일치하는 행 {count}개',
|
|
54
|
+
'safetensors.matchingRowsLimited': '전체 {total}행 중 일치하는 {visible}행 표시',
|
|
51
55
|
'proto.title': 'Protocol Buffer', 'proto.summary': '{syntax} · {package} · 메시지 {messages}개 · 서비스 {services}개',
|
|
52
56
|
'proto.noPackage': '패키지 없음', 'proto.search': '메시지, 필드, enum, RPC 검색', 'proto.copyPanel': '패널 복사',
|
|
53
57
|
'proto.panel.tree': '메시지 트리', 'proto.panel.types': '타입', 'proto.panel.relationships': '관계',
|
|
@@ -149,6 +153,9 @@ export const CATALOG_KO = {
|
|
|
149
153
|
'structured.copyValue': '값', 'structured.copyJson': 'JSON',
|
|
150
154
|
'structured.source': '소스', 'structured.sourceDescription': '편집 가능한 원본 텍스트',
|
|
151
155
|
'structured.treeDescription': '구조화된 미리보기',
|
|
156
|
+
'structured.scopeLabel': '검색 범위', 'structured.scope.all': '전체', 'structured.scope.key': '키',
|
|
157
|
+
'structured.scope.path': '경로', 'structured.scope.value': '값',
|
|
158
|
+
'structured.matchCount': '{count}개 일치', 'structured.keyCount': '키 {count}개', 'structured.itemCount': '항목 {count}개',
|
|
152
159
|
'toml.treeView': 'TOML', 'yaml.treeView': 'YAML', 'jsonl.title': 'JSON Lines',
|
|
153
160
|
'jsonl.loadMore': '더 불러오기', 'jsonl.loadAll': '모두 불러오기', 'jsonl.save': '저장',
|
|
154
161
|
'jsonl.editUnavailable': '편집하려면 전체 파일이 필요합니다.', 'diag.jsonl.load-failed': 'JSON Lines 데이터를 더 불러올 수 없습니다.',
|
|
@@ -328,6 +335,7 @@ export const CATALOG_KO = {
|
|
|
328
335
|
'pdf.goToPage': '페이지 이동',
|
|
329
336
|
'pdf.zoomIn': '확대',
|
|
330
337
|
'pdf.zoomOut': '축소',
|
|
338
|
+
'pdf.thumbnails': '페이지 썸네일',
|
|
331
339
|
'pdf.viewOptions': '보기 옵션',
|
|
332
340
|
'pdf.fitWidth': '너비 맞춤',
|
|
333
341
|
'pdf.fitHeight': '높이 맞춤',
|
|
@@ -369,6 +377,11 @@ export const CATALOG_KO = {
|
|
|
369
377
|
'pdf.textPlace': '페이지를 클릭하여 텍스트를 배치하세요.',
|
|
370
378
|
'pdf.deletePage': '페이지 삭제',
|
|
371
379
|
'pdf.deleteAnnotation': '주석 삭제',
|
|
380
|
+
'pdf.markupList': '주석 목록',
|
|
381
|
+
'pdf.closeMarkupList': '주석 목록 닫기',
|
|
382
|
+
'pdf.changeColor': '색상 변경',
|
|
383
|
+
'pdf.copyText': '텍스트 복사',
|
|
384
|
+
'pdf.noMarkups': '아직 하이라이트, 밑줄, 취소선이 없습니다.',
|
|
372
385
|
'pdf.password': '암호',
|
|
373
386
|
'pdf.passwordRequired': '이 PDF에는 암호가 필요합니다.',
|
|
374
387
|
'pdf.passwordIncorrect': '암호가 올바르지 않습니다. 다시 시도하세요.',
|
|
@@ -40,6 +40,9 @@ export const CATALOG_ZH_CN = {
|
|
|
40
40
|
'hdf5.title': 'HDF5', 'hdf5.search': '搜索表格…', 'hdf5.copyJson': '复制 JSON',
|
|
41
41
|
'hdf5.structure': '结构', 'hdf5.noData': '没有可用数据。',
|
|
42
42
|
'hdf5.matchingRows': '{count} 个匹配行', 'hdf5.matchingRowsLimited': '显示 {visible} / {total} 个匹配行',
|
|
43
|
+
'safetensors.title': 'Safetensors', 'safetensors.search': '搜索张量…', 'safetensors.copyJson': '复制 JSON',
|
|
44
|
+
'safetensors.structure': '结构', 'safetensors.noData': '没有可用数据。',
|
|
45
|
+
'safetensors.matchingRows': '{count} 个匹配行', 'safetensors.matchingRowsLimited': '显示 {visible} / {total} 个匹配行',
|
|
43
46
|
'proto.title': 'Protocol Buffer', 'proto.summary': '{syntax} · {package} · {messages} 条消息 · {services} 个服务',
|
|
44
47
|
'proto.noPackage': '无软件包', 'proto.search': '搜索消息、字段、枚举或 RPC', 'proto.copyPanel': '复制面板',
|
|
45
48
|
'proto.panel.tree': '消息树', 'proto.panel.types': '类型', 'proto.panel.relationships': '关系',
|
|
@@ -141,6 +144,9 @@ export const CATALOG_ZH_CN = {
|
|
|
141
144
|
'structured.copy': '复制', 'structured.copyPath': '路径', 'structured.copyValue': '值', 'structured.copyJson': 'JSON',
|
|
142
145
|
'structured.source': '源代码', 'structured.sourceDescription': '可编辑的原始文本',
|
|
143
146
|
'structured.treeDescription': '结构化预览',
|
|
147
|
+
'structured.scopeLabel': '搜索范围', 'structured.scope.all': '全部', 'structured.scope.key': '键',
|
|
148
|
+
'structured.scope.path': '路径', 'structured.scope.value': '值',
|
|
149
|
+
'structured.matchCount': '{count} 项匹配', 'structured.keyCount': '{count} 个键', 'structured.itemCount': '{count} 个项目',
|
|
144
150
|
'toml.treeView': 'TOML', 'yaml.treeView': 'YAML', 'jsonl.title': 'JSON Lines',
|
|
145
151
|
'jsonl.loadMore': '加载更多', 'jsonl.loadAll': '全部加载', 'jsonl.save': '保存',
|
|
146
152
|
'jsonl.editUnavailable': '编辑需要完整文件。', 'diag.jsonl.load-failed': '无法加载更多 JSON Lines 数据。',
|
|
@@ -227,6 +233,7 @@ export const CATALOG_ZH_CN = {
|
|
|
227
233
|
'json.result.copy': '复制', 'json.result.replace': '替换编辑器内容', 'json.result.close': '关闭',
|
|
228
234
|
'pdf.title': 'PDF', 'pdf.loading': '正在加载 PDF…', 'pdf.loadFailed': '无法加载此 PDF。',
|
|
229
235
|
'pdf.pages': '页', 'pdf.goToPage': '转到页面', 'pdf.zoomIn': '放大', 'pdf.zoomOut': '缩小',
|
|
236
|
+
'pdf.thumbnails': '页面缩略图',
|
|
230
237
|
'pdf.viewOptions': '视图选项', 'pdf.fitWidth': '适合宽度', 'pdf.fitHeight': '适合高度',
|
|
231
238
|
'pdf.singlePage': '单页', 'pdf.twoPagesOdd': '双页(奇数页在左)', 'pdf.twoPagesEven': '双页(偶数页在左)',
|
|
232
239
|
'pdf.matchTheme': '匹配主题颜色', 'pdf.resetPages': '重置页面',
|
|
@@ -245,6 +252,9 @@ export const CATALOG_ZH_CN = {
|
|
|
245
252
|
'pdf.markupOptions': '文本标记选项', 'pdf.underline': '下划线', 'pdf.underlineColor': '下划线颜色',
|
|
246
253
|
'pdf.strikeout': '删除线', 'pdf.strikeoutColor': '删除线颜色', 'pdf.cancel': '取消',
|
|
247
254
|
'pdf.textPlace': '点击页面以放置文本。', 'pdf.deletePage': '删除页面', 'pdf.deleteAnnotation': '删除批注',
|
|
255
|
+
'pdf.markupList': '批注列表', 'pdf.closeMarkupList': '关闭批注列表',
|
|
256
|
+
'pdf.changeColor': '更改颜色', 'pdf.copyText': '复制文本',
|
|
257
|
+
'pdf.noMarkups': '尚无高亮、下划线或删除线。',
|
|
248
258
|
'pdf.password': '密码', 'pdf.passwordRequired': '此 PDF 需要密码。',
|
|
249
259
|
'pdf.passwordIncorrect': '密码错误,请重试。', 'pdf.passwordSubmit': '打开',
|
|
250
260
|
'pdf.editingUnavailable': '此平台无法编辑 PDF', 'pdf.noFilePick': '此平台无法选择要合并的文件',
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
export interface SafetensorsSummaryItem {
|
|
2
|
+
label: string;
|
|
3
|
+
value: string | number;
|
|
4
|
+
}
|
|
5
|
+
export interface SafetensorsTable {
|
|
6
|
+
title: string;
|
|
7
|
+
headers: string[];
|
|
8
|
+
rows: Array<Array<string | number>>;
|
|
9
|
+
}
|
|
10
|
+
export interface SafetensorsDocument {
|
|
11
|
+
format: 'safetensors';
|
|
12
|
+
title: string;
|
|
13
|
+
fileSize: string;
|
|
14
|
+
summary: SafetensorsSummaryItem[];
|
|
15
|
+
tables: SafetensorsTable[];
|
|
16
|
+
rawPreview?: string | undefined;
|
|
17
|
+
warnings: string[];
|
|
18
|
+
}
|
|
19
|
+
/** One tensor as declared in the safetensors header. */
|
|
20
|
+
export interface SafetensorsTensor {
|
|
21
|
+
name: string;
|
|
22
|
+
dtype: string;
|
|
23
|
+
shape: number[];
|
|
24
|
+
/** [begin, end) byte range inside the tensor data buffer. */
|
|
25
|
+
dataOffsets: [number, number];
|
|
26
|
+
/** Product of `shape` (1 for a scalar / empty shape). */
|
|
27
|
+
elements: number;
|
|
28
|
+
/** `end - begin`, the bytes the tensor occupies. */
|
|
29
|
+
byteLength: number;
|
|
30
|
+
}
|
|
31
|
+
export declare function parseSafetensors(input: Uint8Array, fileSize?: string): SafetensorsDocument;
|
|
32
|
+
/** Compact parameter counts, e.g. 1.20B, 350.0M, 12.3K. */
|
|
33
|
+
export declare function formatCount(count: number): string;
|
|
34
|
+
export declare function formatFileSize(bytes: number): string;
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Minimal safetensors (.safetensors) header reader.
|
|
3
|
+
*
|
|
4
|
+
* The format is a fixed 8-byte little-endian header length, followed by a UTF-8
|
|
5
|
+
* JSON header (tensor name → {dtype, shape, data_offsets}, plus an optional
|
|
6
|
+
* "__metadata__" string map), followed by the raw tensor buffer. Only the header
|
|
7
|
+
* is decoded here — the tensor payloads are never read, so inspecting a
|
|
8
|
+
* multi-gigabyte checkpoint stays cheap.
|
|
9
|
+
*
|
|
10
|
+
* Reference: https://github.com/huggingface/safetensors
|
|
11
|
+
*/
|
|
12
|
+
const HEADER_LENGTH_BYTES = 8;
|
|
13
|
+
// A safetensors header is JSON metadata only; real files stay well under this.
|
|
14
|
+
// The cap guards against a corrupt length field claiming gigabytes of "header".
|
|
15
|
+
const MAX_HEADER_BYTES = 100_000_000;
|
|
16
|
+
const METADATA_KEY = '__metadata__';
|
|
17
|
+
/** Bits per element for the dtypes supported by safetensors 0.8. */
|
|
18
|
+
const DTYPE_BITS = {
|
|
19
|
+
F64: 64, C64: 64, I64: 64, U64: 64,
|
|
20
|
+
F32: 32, I32: 32, U32: 32,
|
|
21
|
+
F16: 16, BF16: 16, I16: 16, U16: 16,
|
|
22
|
+
I8: 8, U8: 8, BOOL: 8,
|
|
23
|
+
F8_E4M3: 8, F8_E5M2: 8, F8_E8M0: 8,
|
|
24
|
+
F8_E4M3FNUZ: 8, F8_E5M2FNUZ: 8,
|
|
25
|
+
F6_E2M3: 6, F6_E3M2: 6, F4: 4
|
|
26
|
+
};
|
|
27
|
+
export function parseSafetensors(input, fileSize = formatFileSize(input.byteLength)) {
|
|
28
|
+
const warnings = [];
|
|
29
|
+
if (input.byteLength < HEADER_LENGTH_BYTES) {
|
|
30
|
+
return invalid(fileSize, 'File is too small to contain a safetensors header.');
|
|
31
|
+
}
|
|
32
|
+
const view = new DataView(input.buffer, input.byteOffset, input.byteLength);
|
|
33
|
+
const headerLength = Number(view.getBigUint64(0, true));
|
|
34
|
+
if (!Number.isSafeInteger(headerLength) || headerLength <= 0 || headerLength > MAX_HEADER_BYTES) {
|
|
35
|
+
return invalid(fileSize, 'Header length is out of range; the file is not a valid safetensors file.');
|
|
36
|
+
}
|
|
37
|
+
const headerEnd = HEADER_LENGTH_BYTES + headerLength;
|
|
38
|
+
if (headerEnd > input.byteLength) {
|
|
39
|
+
return invalid(fileSize, 'Declared header length extends past the end of the file.');
|
|
40
|
+
}
|
|
41
|
+
const headerBytes = input.subarray(HEADER_LENGTH_BYTES, headerEnd);
|
|
42
|
+
if (headerBytes[0] !== 0x7b) {
|
|
43
|
+
return invalid(fileSize, 'The safetensors header must begin with a JSON object.');
|
|
44
|
+
}
|
|
45
|
+
let header;
|
|
46
|
+
try {
|
|
47
|
+
const json = new TextDecoder('utf-8', { fatal: true }).decode(headerBytes);
|
|
48
|
+
const parsed = JSON.parse(json);
|
|
49
|
+
if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
|
50
|
+
return invalid(fileSize, 'The safetensors header is not a JSON object.');
|
|
51
|
+
}
|
|
52
|
+
header = parsed;
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
return invalid(fileSize, 'The safetensors header is not valid JSON.');
|
|
56
|
+
}
|
|
57
|
+
const metadata = {};
|
|
58
|
+
const rawMeta = header[METADATA_KEY];
|
|
59
|
+
let metadataIssue = false;
|
|
60
|
+
if (rawMeta !== undefined) {
|
|
61
|
+
if (rawMeta !== null && typeof rawMeta === 'object' && !Array.isArray(rawMeta)) {
|
|
62
|
+
for (const [key, value] of Object.entries(rawMeta)) {
|
|
63
|
+
if (typeof value === 'string')
|
|
64
|
+
metadata[key] = value;
|
|
65
|
+
else
|
|
66
|
+
metadataIssue = true;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
else {
|
|
70
|
+
metadataIssue = true;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
if (metadataIssue)
|
|
74
|
+
warnings.push('Some "__metadata__" values are not strings and were ignored.');
|
|
75
|
+
const dataBufferSize = input.byteLength - headerEnd;
|
|
76
|
+
const tensors = [];
|
|
77
|
+
const dtypeCounts = new Map();
|
|
78
|
+
let totalElements = 0;
|
|
79
|
+
let unknownDtype = false;
|
|
80
|
+
let entryIssue = false;
|
|
81
|
+
let rangeIssue = false;
|
|
82
|
+
const validRanges = [];
|
|
83
|
+
for (const [name, rawValue] of Object.entries(header)) {
|
|
84
|
+
if (name === METADATA_KEY)
|
|
85
|
+
continue;
|
|
86
|
+
const entry = rawValue;
|
|
87
|
+
const dtypeValid = typeof entry?.dtype === 'string';
|
|
88
|
+
const dtype = dtypeValid ? entry.dtype : 'unknown';
|
|
89
|
+
const shapeValid = Array.isArray(entry?.shape)
|
|
90
|
+
&& entry.shape.every(n => Number.isSafeInteger(n) && n >= 0);
|
|
91
|
+
const shape = shapeValid ? entry.shape : [];
|
|
92
|
+
const offsets = Array.isArray(entry?.data_offsets) ? entry.data_offsets : [];
|
|
93
|
+
const offsetsValid = offsets.length === 2
|
|
94
|
+
&& offsets.every(n => Number.isSafeInteger(n) && n >= 0);
|
|
95
|
+
const begin = offsetsValid ? offsets[0] : 0;
|
|
96
|
+
const end = offsetsValid ? offsets[1] : 0;
|
|
97
|
+
if (!dtypeValid || !shapeValid || !offsetsValid)
|
|
98
|
+
entryIssue = true;
|
|
99
|
+
const elements = safeElementCount(shape);
|
|
100
|
+
if (elements === undefined)
|
|
101
|
+
entryIssue = true;
|
|
102
|
+
const elementCount = elements ?? 0;
|
|
103
|
+
const byteLength = Math.max(0, end - begin);
|
|
104
|
+
tensors.push({ name, dtype, shape, dataOffsets: [begin, end], elements: elementCount, byteLength });
|
|
105
|
+
totalElements += elementCount;
|
|
106
|
+
dtypeCounts.set(dtype, (dtypeCounts.get(dtype) ?? 0) + 1);
|
|
107
|
+
const bits = DTYPE_BITS[dtype];
|
|
108
|
+
if (bits === undefined) {
|
|
109
|
+
unknownDtype = true;
|
|
110
|
+
}
|
|
111
|
+
else if (elements !== undefined) {
|
|
112
|
+
const expectedBits = BigInt(elements) * BigInt(bits);
|
|
113
|
+
if (expectedBits % 8n !== 0n || expectedBits / 8n !== BigInt(byteLength))
|
|
114
|
+
rangeIssue = true;
|
|
115
|
+
}
|
|
116
|
+
if (!offsetsValid || end < begin || end > dataBufferSize) {
|
|
117
|
+
rangeIssue = true;
|
|
118
|
+
}
|
|
119
|
+
else {
|
|
120
|
+
validRanges.push([begin, end]);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
// Header keys have no defined order; sort tensors by their data offset so the
|
|
124
|
+
// table reads in storage order.
|
|
125
|
+
tensors.sort((a, b) => a.dataOffsets[0] - b.dataOffsets[0] || a.dataOffsets[1] - b.dataOffsets[1]);
|
|
126
|
+
// A valid safetensors data buffer is packed from byte zero to EOF without
|
|
127
|
+
// holes or overlaps. Empty tensors may repeat the current offset.
|
|
128
|
+
validRanges.sort((a, b) => a[0] - b[0] || a[1] - b[1]);
|
|
129
|
+
let coveredUntil = 0;
|
|
130
|
+
for (const [begin, end] of validRanges) {
|
|
131
|
+
if (begin !== coveredUntil)
|
|
132
|
+
rangeIssue = true;
|
|
133
|
+
coveredUntil = Math.max(coveredUntil, end);
|
|
134
|
+
}
|
|
135
|
+
if (coveredUntil !== dataBufferSize)
|
|
136
|
+
rangeIssue = true;
|
|
137
|
+
if (entryIssue)
|
|
138
|
+
warnings.push('Some tensor entries have an invalid dtype, shape, or data offset declaration.');
|
|
139
|
+
if (unknownDtype)
|
|
140
|
+
warnings.push('Some tensors use a dtype this viewer does not recognize.');
|
|
141
|
+
if (rangeIssue)
|
|
142
|
+
warnings.push('Some tensor byte ranges are inconsistent with their dtype and shape.');
|
|
143
|
+
const dtypeList = [...dtypeCounts.keys()].sort();
|
|
144
|
+
const summary = [
|
|
145
|
+
{ label: 'Tensors', value: tensors.length },
|
|
146
|
+
{ label: 'Parameters', value: formatCount(totalElements) },
|
|
147
|
+
{ label: 'Data types', value: dtypeList.length ? dtypeList.join(', ') : '—' }
|
|
148
|
+
];
|
|
149
|
+
if (Object.keys(metadata).length) {
|
|
150
|
+
summary.push({ label: 'Metadata keys', value: Object.keys(metadata).length });
|
|
151
|
+
}
|
|
152
|
+
const tables = [
|
|
153
|
+
{
|
|
154
|
+
title: `Tensors (${tensors.length})`,
|
|
155
|
+
headers: ['Name', 'Dtype', 'Shape', 'Parameters', 'Size'],
|
|
156
|
+
rows: tensors.map(tensor => [
|
|
157
|
+
tensor.name,
|
|
158
|
+
tensor.dtype,
|
|
159
|
+
tensor.shape.length ? tensor.shape.join(' × ') : 'scalar',
|
|
160
|
+
tensor.elements,
|
|
161
|
+
formatFileSize(tensor.byteLength)
|
|
162
|
+
])
|
|
163
|
+
}
|
|
164
|
+
];
|
|
165
|
+
if (Object.keys(metadata).length) {
|
|
166
|
+
tables.push({
|
|
167
|
+
title: `Metadata (${Object.keys(metadata).length})`,
|
|
168
|
+
headers: ['Key', 'Value'],
|
|
169
|
+
rows: Object.entries(metadata).map(([key, value]) => [key, value])
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
const rawPreview = tensors.length
|
|
173
|
+
? tensors
|
|
174
|
+
.map(tensor => `${tensor.name} [${tensor.shape.length ? tensor.shape.join(' × ') : 'scalar'}] ${tensor.dtype}`)
|
|
175
|
+
.join('\n')
|
|
176
|
+
: undefined;
|
|
177
|
+
return {
|
|
178
|
+
format: 'safetensors',
|
|
179
|
+
title: 'Safetensors tensor collection',
|
|
180
|
+
fileSize,
|
|
181
|
+
summary,
|
|
182
|
+
tables,
|
|
183
|
+
rawPreview,
|
|
184
|
+
warnings
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
function safeElementCount(shape) {
|
|
188
|
+
let count = 1;
|
|
189
|
+
for (const dimension of shape) {
|
|
190
|
+
if (!Number.isSafeInteger(dimension) || dimension < 0)
|
|
191
|
+
return undefined;
|
|
192
|
+
if (dimension !== 0 && count > Number.MAX_SAFE_INTEGER / dimension)
|
|
193
|
+
return undefined;
|
|
194
|
+
count *= dimension;
|
|
195
|
+
}
|
|
196
|
+
return count;
|
|
197
|
+
}
|
|
198
|
+
function invalid(fileSize, message) {
|
|
199
|
+
return {
|
|
200
|
+
format: 'safetensors',
|
|
201
|
+
title: 'Safetensors tensor collection',
|
|
202
|
+
fileSize,
|
|
203
|
+
summary: [{ label: 'Status', value: 'invalid' }],
|
|
204
|
+
tables: [],
|
|
205
|
+
rawPreview: undefined,
|
|
206
|
+
warnings: [message]
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
/** Compact parameter counts, e.g. 1.20B, 350.0M, 12.3K. */
|
|
210
|
+
export function formatCount(count) {
|
|
211
|
+
if (count < 1000)
|
|
212
|
+
return String(count);
|
|
213
|
+
const units = [[1e9, 'B'], [1e6, 'M'], [1e3, 'K']];
|
|
214
|
+
for (const [threshold, suffix] of units) {
|
|
215
|
+
if (count >= threshold) {
|
|
216
|
+
const value = count / threshold;
|
|
217
|
+
return value.toFixed(value >= 100 ? 0 : value >= 10 ? 1 : 2) + suffix;
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
return String(count);
|
|
221
|
+
}
|
|
222
|
+
export function formatFileSize(bytes) {
|
|
223
|
+
if (bytes < 1024)
|
|
224
|
+
return String(bytes) + ' bytes';
|
|
225
|
+
const units = ['KB', 'MB', 'GB', 'TB'];
|
|
226
|
+
let value = bytes / 1024;
|
|
227
|
+
let unit = units[0];
|
|
228
|
+
for (let i = 1; i < units.length && value >= 1024; i++) {
|
|
229
|
+
value /= 1024;
|
|
230
|
+
unit = units[i];
|
|
231
|
+
}
|
|
232
|
+
return value.toFixed(value >= 10 ? 1 : 2) + ' ' + unit;
|
|
233
|
+
}
|
|
@@ -1,4 +1,10 @@
|
|
|
1
1
|
export type TomlValueKind = 'table' | 'array' | 'string' | 'integer' | 'float' | 'boolean' | 'datetime';
|
|
2
|
+
/** Half-open character offsets into `TomlDocument.text`, so a host can map a
|
|
3
|
+
* node onto a text selection and a caret offset back onto a node. */
|
|
4
|
+
export interface TomlRange {
|
|
5
|
+
start: number;
|
|
6
|
+
end: number;
|
|
7
|
+
}
|
|
2
8
|
export interface TomlNode {
|
|
3
9
|
kind: TomlValueKind;
|
|
4
10
|
key: string;
|
|
@@ -6,6 +12,10 @@ export interface TomlNode {
|
|
|
6
12
|
value?: string | boolean;
|
|
7
13
|
raw: string;
|
|
8
14
|
line: number;
|
|
15
|
+
/** `#` comments attached to the node: the run directly above it plus any
|
|
16
|
+
* trailing comment on the declaration line, `#` and padding removed. */
|
|
17
|
+
comment?: string;
|
|
18
|
+
range?: TomlRange;
|
|
9
19
|
children?: TomlNode[];
|
|
10
20
|
}
|
|
11
21
|
export interface TomlDocument {
|