@qvac/core 0.11.2 → 0.11.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +49 -1
- package/dist-lib/lib/assistant/history.js +1 -1
- package/dist-lib/lib/assistant/message.d.ts +4 -1
- package/dist-lib/lib/assistant/pdf-budget.d.ts +10 -2
- package/dist-lib/lib/assistant/pdf-budget.js +10 -11
- package/dist-lib/lib/assistant/pdf-pages.d.ts +18 -0
- package/dist-lib/lib/assistant/pdf-pages.js +75 -0
- package/dist-lib/lib/assistant.d.ts +5 -5
- package/dist-lib/lib/assistant.js +144 -122
- package/dist-lib/lib/core/document-attachment.d.ts +6 -3
- package/dist-lib/lib/core/document-attachment.js +10 -5
- package/dist-lib/lib/core/files/index.d.ts +1 -1
- package/dist-lib/lib/core/files/index.js +1 -1
- package/dist-lib/lib/core/files/rag/index.d.ts +1 -1
- package/dist-lib/lib/core/files/rag/index.js +1 -1
- package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +1 -0
- package/dist-lib/lib/core/files/rag/retrieval/full-text.js +3 -0
- package/dist-lib/lib/core/mesh.d.ts +3 -0
- package/dist-lib/lib/core/mesh.js +55 -17
- package/dist-lib/lib/core/migrations/file-index.d.ts +1 -1
- package/dist-lib/lib/core/migrations/file-index.js +6 -2
- package/dist-lib/lib/core/migrations/index.d.ts +2 -1
- package/dist-lib/lib/core/migrations/index.js +65 -8
- package/dist-lib/lib/core/pdf/figure.d.ts +22 -0
- package/dist-lib/lib/core/pdf/figure.js +64 -0
- package/dist-lib/lib/core/pdf/rasterize.d.ts +10 -1
- package/dist-lib/lib/core/pdf/rasterize.js +72 -22
- package/dist-lib/lib/core.js +21 -3
- package/package.json +2 -2
|
@@ -8,7 +8,7 @@ import crypto from 'hypercore-crypto';
|
|
|
8
8
|
import ReadyResource from 'ready-resource';
|
|
9
9
|
import { Files } from './files/index.js';
|
|
10
10
|
import MeshDefinition from '../../spec/mesh/hyperdb/index.js';
|
|
11
|
-
import { Router, encode } from '../../spec/mesh/hyperdispatch/index.js';
|
|
11
|
+
import { Router, encode, version } from '../../spec/mesh/hyperdispatch/index.js';
|
|
12
12
|
import { syncChanges } from './sync/index.js';
|
|
13
13
|
import { ViewUpdates } from './sync/view-updates.js';
|
|
14
14
|
import { buildDefaultSeed, isOwnKickedRow, isOwnRevokedRow } from './helpers/index.js';
|
|
@@ -25,6 +25,7 @@ import { errorMessage, isRecord } from '../guards.js';
|
|
|
25
25
|
import { CoreError } from '../errors.js';
|
|
26
26
|
import { ACTIVE_AGENTS, ACTIVE_CHATS, DEVICES, LIVE_WRITERS_BY_MOST_RECENT, MEMBERS, MESH_NAMES } from './collections.js';
|
|
27
27
|
const [MESH_KEYPAIR, MESH_ENCRYPTION, MESH_DEVICE] = crypto.namespace('qvac-assistant/mesh', 3);
|
|
28
|
+
export const SCHEMA_VERSION = version;
|
|
28
29
|
const SLOW_WRITE_MS = 5000;
|
|
29
30
|
const SLOW_DRAIN_MS = 15_000;
|
|
30
31
|
export function meshPublicKeyFromSeed(seed) {
|
|
@@ -264,6 +265,7 @@ export class Mesh extends ReadyResource {
|
|
|
264
265
|
async _open() {
|
|
265
266
|
this._openedAt = Date.now();
|
|
266
267
|
this._log?.info(this._scope, 'opening', {
|
|
268
|
+
schema: SCHEMA_VERSION,
|
|
267
269
|
creator: this._key === null,
|
|
268
270
|
compute: this._compute,
|
|
269
271
|
bootFrom: this._bootFrom?.length ?? null,
|
|
@@ -461,25 +463,33 @@ export class Mesh extends ReadyResource {
|
|
|
461
463
|
}
|
|
462
464
|
async _watchStall(pending, watch) {
|
|
463
465
|
const started = Date.now();
|
|
464
|
-
|
|
465
|
-
const
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
466
|
+
const clock = { reports: 0, timer: null };
|
|
467
|
+
const report = (wait) => {
|
|
468
|
+
clock.timer = setTimeout(() => {
|
|
469
|
+
if (!this.closing) {
|
|
470
|
+
report(wait * 2);
|
|
471
|
+
}
|
|
472
|
+
if (clock.reports === 0 && this._stallReported) {
|
|
473
|
+
return;
|
|
474
|
+
}
|
|
475
|
+
this._stallReported = true;
|
|
476
|
+
clock.reports++;
|
|
477
|
+
this._reportStall(watch, { started, reports: clock.reports }).catch(noop);
|
|
478
|
+
}, wait);
|
|
479
|
+
};
|
|
480
|
+
report(watch.ms);
|
|
473
481
|
try {
|
|
474
482
|
await pending;
|
|
475
|
-
this._endStall(watch, { reported, started, failed: false });
|
|
483
|
+
this._endStall(watch, { reported: clock.reports > 0, started, failed: false });
|
|
476
484
|
}
|
|
477
485
|
catch (err) {
|
|
478
|
-
this._endStall(watch, { reported, started, failed: true, err });
|
|
486
|
+
this._endStall(watch, { reported: clock.reports > 0, started, failed: true, err });
|
|
479
487
|
throw err;
|
|
480
488
|
}
|
|
481
489
|
finally {
|
|
482
|
-
|
|
490
|
+
if (clock.timer) {
|
|
491
|
+
clearTimeout(clock.timer);
|
|
492
|
+
}
|
|
483
493
|
}
|
|
484
494
|
}
|
|
485
495
|
_endStall({ what, op }, { reported, started, failed, err }) {
|
|
@@ -495,23 +505,39 @@ export class Mesh extends ReadyResource {
|
|
|
495
505
|
this._log?.info(this._scope, `${what} settled`, { op, ms });
|
|
496
506
|
}
|
|
497
507
|
}
|
|
498
|
-
async _reportStall({ what, op }, started) {
|
|
499
|
-
const { stats, system } = this.autobee;
|
|
508
|
+
async _reportStall({ what, op }, { started, reports }) {
|
|
509
|
+
const { stats, system, writable } = this.autobee;
|
|
500
510
|
const missing = await this._coresNotHeld([system.bee, this.autobee.bee]).catch(noop);
|
|
511
|
+
const fastForwarding = await this._fastForwardHeld().catch(noop);
|
|
512
|
+
const writers = writable ? undefined : await this._writersHeld().catch(noop);
|
|
501
513
|
this._log?.warn(this._scope, `${what} still pending`, {
|
|
502
514
|
op,
|
|
515
|
+
reports,
|
|
503
516
|
afterMs: Date.now() - started,
|
|
504
|
-
writable
|
|
517
|
+
writable,
|
|
505
518
|
interrupted: !!this.autobee.interrupted,
|
|
506
519
|
drains: stats.drains,
|
|
507
520
|
applies: stats.applies,
|
|
508
521
|
appends: stats.appends,
|
|
509
522
|
fastForwards: stats.fastForwards,
|
|
523
|
+
fastForwarding,
|
|
510
524
|
localLength: this.autobee.local.length,
|
|
511
525
|
missing,
|
|
526
|
+
writers,
|
|
512
527
|
blindPeers: this.blindPeerStats()
|
|
513
528
|
});
|
|
514
529
|
}
|
|
530
|
+
async _fastForwardHeld() {
|
|
531
|
+
const ff = this.autobee.fastForwarding;
|
|
532
|
+
if (!ff) {
|
|
533
|
+
return null;
|
|
534
|
+
}
|
|
535
|
+
return { to: ff.head.length, ...(await this._coreHeld(ff.head.key)) };
|
|
536
|
+
}
|
|
537
|
+
async _writersHeld() {
|
|
538
|
+
const writers = [...this.autobee.activeWriters];
|
|
539
|
+
return Promise.all(writers.map((writer) => this._coreHeld(writer.core.key)));
|
|
540
|
+
}
|
|
515
541
|
async _coresNotHeld(views) {
|
|
516
542
|
const keys = new Map();
|
|
517
543
|
for (const { context } of views) {
|
|
@@ -526,11 +552,14 @@ export class Mesh extends ReadyResource {
|
|
|
526
552
|
const core = this._store.get({ key, active: false });
|
|
527
553
|
try {
|
|
528
554
|
await core.ready();
|
|
555
|
+
const reach = core.peers.map(remoteReach);
|
|
529
556
|
return {
|
|
530
557
|
key: b4a.toHex(key).slice(0, 8),
|
|
531
558
|
length: core.length,
|
|
532
559
|
held: core.contiguousLength,
|
|
533
|
-
peers: core.peers.length
|
|
560
|
+
peers: core.peers.length,
|
|
561
|
+
peerLength: Math.max(0, ...reach.map((peer) => peer.length)),
|
|
562
|
+
peerHeld: Math.max(0, ...reach.map((peer) => peer.held))
|
|
534
563
|
};
|
|
535
564
|
}
|
|
536
565
|
finally {
|
|
@@ -951,6 +980,15 @@ export class MeshBatch {
|
|
|
951
980
|
}
|
|
952
981
|
}
|
|
953
982
|
function noop() { }
|
|
983
|
+
function remoteReach(peer) {
|
|
984
|
+
if (!isRecord(peer)) {
|
|
985
|
+
return { length: 0, held: 0 };
|
|
986
|
+
}
|
|
987
|
+
return {
|
|
988
|
+
length: typeof peer.remoteLength === 'number' ? peer.remoteLength : 0,
|
|
989
|
+
held: typeof peer.remoteContiguousLength === 'number' ? peer.remoteContiguousLength : 0
|
|
990
|
+
};
|
|
991
|
+
}
|
|
954
992
|
function viewCoreOf(head) {
|
|
955
993
|
return isRecord(head) && b4a.isBuffer(head.key) ? b4a.toHex(head.key).slice(0, 8) : null;
|
|
956
994
|
}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
import type { Mesh } from '../mesh.ts';
|
|
2
|
-
export declare function backfillFileIndex(mesh: Mesh): Promise<
|
|
2
|
+
export declare function backfillFileIndex(mesh: Mesh): Promise<number>;
|
|
@@ -3,26 +3,30 @@ const BATCH = 256;
|
|
|
3
3
|
export async function backfillFileIndex(mesh) {
|
|
4
4
|
const index = mesh.fileIndex;
|
|
5
5
|
if (!index) {
|
|
6
|
-
return;
|
|
6
|
+
return 0;
|
|
7
7
|
}
|
|
8
8
|
const snapshot = mesh.view.snapshot();
|
|
9
|
+
let recorded = 0;
|
|
9
10
|
try {
|
|
10
11
|
let batch = [];
|
|
11
12
|
for await (const chunk of attachmentChunks(snapshot)) {
|
|
12
13
|
if (mesh.closing)
|
|
13
|
-
return;
|
|
14
|
+
return recorded;
|
|
14
15
|
batch.push(chunk);
|
|
15
16
|
if (batch.length < BATCH) {
|
|
16
17
|
continue;
|
|
17
18
|
}
|
|
18
19
|
await index.record(batch);
|
|
20
|
+
recorded += batch.length;
|
|
19
21
|
batch = [];
|
|
20
22
|
}
|
|
21
23
|
if (batch.length > 0) {
|
|
22
24
|
await index.record(batch);
|
|
25
|
+
recorded += batch.length;
|
|
23
26
|
}
|
|
24
27
|
}
|
|
25
28
|
finally {
|
|
26
29
|
await snapshot.close();
|
|
27
30
|
}
|
|
31
|
+
return recorded;
|
|
28
32
|
}
|
|
@@ -4,5 +4,6 @@ import { type MembershipMigrationContext } from './membership.ts';
|
|
|
4
4
|
export type MigrationContext = MembershipMigrationContext & AgentCatalogMigrationContext;
|
|
5
5
|
export interface RunMigrationsOptions {
|
|
6
6
|
readonly catalog?: boolean;
|
|
7
|
+
readonly slowMs?: number;
|
|
7
8
|
}
|
|
8
|
-
export declare function runMigrations(mesh: Mesh, context: MigrationContext, { catalog }?: RunMigrationsOptions): Promise<boolean>;
|
|
9
|
+
export declare function runMigrations(mesh: Mesh, context: MigrationContext, { catalog, slowMs }?: RunMigrationsOptions): Promise<boolean>;
|
|
@@ -1,15 +1,72 @@
|
|
|
1
|
+
import b4a from 'b4a';
|
|
1
2
|
import { migrateAgentCatalog } from './agent-catalog.js';
|
|
2
3
|
import { migrateAgentToolsScope } from './agent-tools-scope.js';
|
|
3
4
|
import { migrateMembership } from './membership.js';
|
|
4
5
|
import { sealDeviceRows } from './seal.js';
|
|
5
6
|
import { reindexAgents } from './agent-indexes.js';
|
|
6
7
|
import { reindexDevices } from './device-indexes.js';
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
const
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
8
|
+
const SLOW_MIGRATIONS_MS = 15_000;
|
|
9
|
+
export async function runMigrations(mesh, context, { catalog = true, slowMs = SLOW_MIGRATIONS_MS } = {}) {
|
|
10
|
+
const steps = [
|
|
11
|
+
{ name: 'reindex-devices', migrate: () => reindexDevices(mesh) },
|
|
12
|
+
{ name: 'membership', migrate: () => migrateMembership(mesh, context) },
|
|
13
|
+
...(catalog
|
|
14
|
+
? [{ name: 'agent-catalog', migrate: () => migrateAgentCatalog(mesh, context) }]
|
|
15
|
+
: []),
|
|
16
|
+
{ name: 'seal', migrate: () => sealDeviceRows(mesh, context) },
|
|
17
|
+
{ name: 'reindex-agents', migrate: () => reindexAgents(mesh) },
|
|
18
|
+
{ name: 'agent-tools-scope', migrate: () => migrateAgentToolsScope(mesh) }
|
|
19
|
+
];
|
|
20
|
+
return (await runLogged(mesh, steps, { log: context.log, wait: slowMs })).length > 0;
|
|
21
|
+
}
|
|
22
|
+
async function runLogged(mesh, steps, report) {
|
|
23
|
+
const run = {
|
|
24
|
+
mesh: b4a.toHex(mesh.key).slice(0, 8),
|
|
25
|
+
started: Date.now(),
|
|
26
|
+
step: null,
|
|
27
|
+
timer: null,
|
|
28
|
+
wrote: [],
|
|
29
|
+
ms: {}
|
|
30
|
+
};
|
|
31
|
+
reportWhileSlow(mesh, run, report);
|
|
32
|
+
try {
|
|
33
|
+
await runSteps(steps, run);
|
|
34
|
+
}
|
|
35
|
+
finally {
|
|
36
|
+
if (run.timer) {
|
|
37
|
+
clearTimeout(run.timer);
|
|
38
|
+
}
|
|
39
|
+
report.log?.info(['p2p', 'mesh'], 'migrations ran', {
|
|
40
|
+
mesh: run.mesh,
|
|
41
|
+
ms: Date.now() - run.started,
|
|
42
|
+
wrote: run.wrote,
|
|
43
|
+
steps: run.ms,
|
|
44
|
+
failed: run.step
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
return run.wrote;
|
|
48
|
+
}
|
|
49
|
+
async function runSteps(steps, run) {
|
|
50
|
+
for (const { name, migrate } of steps) {
|
|
51
|
+
run.step = name;
|
|
52
|
+
const at = Date.now();
|
|
53
|
+
if (await migrate()) {
|
|
54
|
+
run.wrote.push(name);
|
|
55
|
+
}
|
|
56
|
+
run.ms[name] = Date.now() - at;
|
|
57
|
+
}
|
|
58
|
+
run.step = null;
|
|
59
|
+
}
|
|
60
|
+
function reportWhileSlow(mesh, run, { log, wait }) {
|
|
61
|
+
run.timer = setTimeout(() => {
|
|
62
|
+
if (mesh.closing) {
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
65
|
+
log?.warn(['p2p', 'mesh'], 'migrations still running', {
|
|
66
|
+
mesh: run.mesh,
|
|
67
|
+
step: run.step,
|
|
68
|
+
afterMs: Date.now() - run.started
|
|
69
|
+
});
|
|
70
|
+
reportWhileSlow(mesh, run, { log, wait: wait * 2 });
|
|
71
|
+
}, wait);
|
|
15
72
|
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
export interface PdfBounds {
|
|
2
|
+
readonly x: number;
|
|
3
|
+
readonly y: number;
|
|
4
|
+
readonly width: number;
|
|
5
|
+
readonly height: number;
|
|
6
|
+
}
|
|
7
|
+
export interface PdfTextSpan extends PdfBounds {
|
|
8
|
+
readonly text: string;
|
|
9
|
+
}
|
|
10
|
+
export interface PdfFigure {
|
|
11
|
+
readonly clip: PdfBounds;
|
|
12
|
+
readonly names: ReadonlyArray<string>;
|
|
13
|
+
}
|
|
14
|
+
export interface PdfPageLayout {
|
|
15
|
+
readonly size: {
|
|
16
|
+
readonly width: number;
|
|
17
|
+
readonly height: number;
|
|
18
|
+
};
|
|
19
|
+
readonly images: ReadonlyArray<PdfBounds>;
|
|
20
|
+
readonly spans: ReadonlyArray<PdfTextSpan>;
|
|
21
|
+
}
|
|
22
|
+
export declare function figureRegion({ size, images, spans }: PdfPageLayout): PdfFigure | null;
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
const CAPTION = /^(?:Figure|Fig\.|Table|Chart|Exhibit)\s+[\w.-]*\w/i;
|
|
2
|
+
const CAPTION_GAP = 72;
|
|
3
|
+
const DECORATION_SHARE = 0.005;
|
|
4
|
+
const WHOLE_PAGE_SHARE = 0.6;
|
|
5
|
+
const MARGIN = 4;
|
|
6
|
+
function boxOf({ x, y, width, height }) {
|
|
7
|
+
return { left: x, bottom: y, right: x + width, top: y + height };
|
|
8
|
+
}
|
|
9
|
+
function join(a, b) {
|
|
10
|
+
return {
|
|
11
|
+
left: Math.min(a.left, b.left),
|
|
12
|
+
bottom: Math.min(a.bottom, b.bottom),
|
|
13
|
+
right: Math.max(a.right, b.right),
|
|
14
|
+
top: Math.max(a.top, b.top)
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
function overlapsAcross(span, box) {
|
|
18
|
+
return span.left < box.right && span.right > box.left;
|
|
19
|
+
}
|
|
20
|
+
function captionBlock(caption, lines, figure) {
|
|
21
|
+
const height = caption.top - caption.bottom;
|
|
22
|
+
let block = caption;
|
|
23
|
+
const below = lines
|
|
24
|
+
.filter((line) => line.top <= caption.top + 1 && line.bottom < caption.bottom)
|
|
25
|
+
.filter((line) => overlapsAcross(line, figure))
|
|
26
|
+
.sort((a, b) => b.top - a.top);
|
|
27
|
+
for (const line of below) {
|
|
28
|
+
if (block.bottom - line.top > height)
|
|
29
|
+
break;
|
|
30
|
+
block = join(block, line);
|
|
31
|
+
}
|
|
32
|
+
return block;
|
|
33
|
+
}
|
|
34
|
+
function captionName(text, line, figure) {
|
|
35
|
+
const name = text.trim().match(CAPTION)?.[0];
|
|
36
|
+
const near = line.top >= figure.bottom - CAPTION_GAP && line.bottom <= figure.top + CAPTION_GAP;
|
|
37
|
+
return name && near && overlapsAcross(line, figure) ? name : null;
|
|
38
|
+
}
|
|
39
|
+
export function figureRegion({ size, images, spans }) {
|
|
40
|
+
const area = size.width * size.height;
|
|
41
|
+
const content = images.filter((image) => image.width * image.height >= DECORATION_SHARE * area);
|
|
42
|
+
const pictures = (content.length ? content : images).map(boxOf);
|
|
43
|
+
if (!pictures.length) {
|
|
44
|
+
return null;
|
|
45
|
+
}
|
|
46
|
+
let box = pictures.reduce(join);
|
|
47
|
+
const figure = box;
|
|
48
|
+
const lines = spans.map(boxOf);
|
|
49
|
+
const names = [];
|
|
50
|
+
for (const [index, span] of spans.entries()) {
|
|
51
|
+
const name = captionName(span.text, lines[index], figure);
|
|
52
|
+
if (name) {
|
|
53
|
+
names.push(name);
|
|
54
|
+
box = join(box, captionBlock(lines[index], lines, figure));
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
const clip = {
|
|
58
|
+
x: Math.max(0, box.left - MARGIN),
|
|
59
|
+
y: Math.max(0, box.bottom - MARGIN),
|
|
60
|
+
width: Math.min(size.width, box.right + MARGIN) - Math.max(0, box.left - MARGIN),
|
|
61
|
+
height: Math.min(size.height, box.top + MARGIN) - Math.max(0, box.bottom - MARGIN)
|
|
62
|
+
};
|
|
63
|
+
return clip.width * clip.height >= WHOLE_PAGE_SHARE * area ? null : { clip, names };
|
|
64
|
+
}
|
|
@@ -1,14 +1,23 @@
|
|
|
1
1
|
import type { LoggerInterface } from '@qvac/logging';
|
|
2
|
+
import { type PdfBounds, type PdfFigure } from './figure.ts';
|
|
2
3
|
export declare const PDF_RENDER_SCALE = 2;
|
|
3
4
|
interface PageFlags {
|
|
4
5
|
hasImage: boolean;
|
|
5
6
|
hasText: boolean;
|
|
6
7
|
}
|
|
8
|
+
interface PageSize {
|
|
9
|
+
width: number;
|
|
10
|
+
height: number;
|
|
11
|
+
}
|
|
7
12
|
export declare function renderScale(pageWidth: number): number;
|
|
13
|
+
export declare function renderSize(page: PageSize, clip?: PdfBounds): PageSize;
|
|
8
14
|
export interface PdfDoc {
|
|
9
15
|
pageCount(): number;
|
|
10
16
|
pageFlags(page: number): PageFlags;
|
|
11
|
-
|
|
17
|
+
figure(page: number): PdfFigure | null;
|
|
18
|
+
blank(page: number): Promise<boolean>;
|
|
19
|
+
renderSize(page: number, clip?: PdfBounds): PageSize;
|
|
20
|
+
renderPage(page: number, clip?: PdfBounds): Promise<Uint8Array>;
|
|
12
21
|
close(): void;
|
|
13
22
|
}
|
|
14
23
|
export interface PdfRasterizer {
|
|
@@ -1,8 +1,30 @@
|
|
|
1
1
|
import { MAX_IMAGE_WIDTH, rasterToModelJpeg } from '../image/normalize.js';
|
|
2
|
+
import { figureRegion } from './figure.js';
|
|
2
3
|
export const PDF_RENDER_SCALE = 2;
|
|
4
|
+
const BLANK_SCALE = 0.25;
|
|
5
|
+
const PAPER = 230;
|
|
6
|
+
const INK_SHARE = 0.001;
|
|
3
7
|
export function renderScale(pageWidth) {
|
|
4
8
|
return Math.min(PDF_RENDER_SCALE, MAX_IMAGE_WIDTH / pageWidth);
|
|
5
9
|
}
|
|
10
|
+
export function renderSize(page, clip) {
|
|
11
|
+
const scale = renderScale(page.width);
|
|
12
|
+
const width = Math.max(1, Math.round((clip?.width ?? page.width) * scale));
|
|
13
|
+
const height = Math.max(1, Math.round((clip?.height ?? page.height) * scale));
|
|
14
|
+
const shrink = Math.min(1, MAX_IMAGE_WIDTH / width);
|
|
15
|
+
return { width: Math.round(width * shrink), height: Math.round(height * shrink) };
|
|
16
|
+
}
|
|
17
|
+
function blankRaster({ data }) {
|
|
18
|
+
let samples = 0;
|
|
19
|
+
let ink = 0;
|
|
20
|
+
for (let i = 0; i < data.length; i += 4) {
|
|
21
|
+
samples++;
|
|
22
|
+
if (data[i] < PAPER || data[i + 1] < PAPER || data[i + 2] < PAPER) {
|
|
23
|
+
ink++;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
return ink <= samples * INK_SHARE;
|
|
27
|
+
}
|
|
6
28
|
let pdfium;
|
|
7
29
|
async function loadPdfium(log) {
|
|
8
30
|
if (pdfium !== undefined)
|
|
@@ -17,6 +39,55 @@ async function loadPdfium(log) {
|
|
|
17
39
|
}
|
|
18
40
|
return pdfium;
|
|
19
41
|
}
|
|
42
|
+
function pageFigure(doc, page) {
|
|
43
|
+
if (doc.pageRotation(page) !== 0) {
|
|
44
|
+
return null;
|
|
45
|
+
}
|
|
46
|
+
const images = doc.imageBounds(page);
|
|
47
|
+
if (!images.length) {
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
50
|
+
const box = doc.pageBox(page);
|
|
51
|
+
const shown = (at) => ({ ...at, x: at.x - box.x, y: at.y - box.y });
|
|
52
|
+
return figureRegion({
|
|
53
|
+
size: { width: box.width, height: box.height },
|
|
54
|
+
images: images.map(shown),
|
|
55
|
+
spans: doc.textSpans(page).map(shown)
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
function wrapDoc(doc) {
|
|
59
|
+
return {
|
|
60
|
+
pageCount() {
|
|
61
|
+
return doc.pageCount();
|
|
62
|
+
},
|
|
63
|
+
pageFlags(page) {
|
|
64
|
+
return doc.pageFlags(page);
|
|
65
|
+
},
|
|
66
|
+
figure(page) {
|
|
67
|
+
return pageFigure(doc, page);
|
|
68
|
+
},
|
|
69
|
+
async blank(page) {
|
|
70
|
+
return blankRaster(await doc.renderAsync(page, { scale: BLANK_SCALE }));
|
|
71
|
+
},
|
|
72
|
+
renderSize(page, clip) {
|
|
73
|
+
return renderSize(doc.pageSize(page), clip);
|
|
74
|
+
},
|
|
75
|
+
async renderPage(page, clip) {
|
|
76
|
+
const scale = renderScale(doc.pageSize(page).width);
|
|
77
|
+
const jpeg = await rasterToModelJpeg(await doc.renderAsync(page, { scale, clip }));
|
|
78
|
+
if (!jpeg) {
|
|
79
|
+
throw new Error('no JPEG encoder: bare-media did not load');
|
|
80
|
+
}
|
|
81
|
+
return jpeg;
|
|
82
|
+
},
|
|
83
|
+
close() {
|
|
84
|
+
try {
|
|
85
|
+
doc.close();
|
|
86
|
+
}
|
|
87
|
+
catch { }
|
|
88
|
+
}
|
|
89
|
+
};
|
|
90
|
+
}
|
|
20
91
|
export const createPdfRasterizer = (log) => ({
|
|
21
92
|
async open(bytes) {
|
|
22
93
|
const mod = await loadPdfium(log);
|
|
@@ -30,27 +101,6 @@ export const createPdfRasterizer = (log) => ({
|
|
|
30
101
|
catch {
|
|
31
102
|
return null;
|
|
32
103
|
}
|
|
33
|
-
return
|
|
34
|
-
pageCount() {
|
|
35
|
-
return doc.pageCount();
|
|
36
|
-
},
|
|
37
|
-
pageFlags(page) {
|
|
38
|
-
return doc.pageFlags(page);
|
|
39
|
-
},
|
|
40
|
-
async renderPage(page) {
|
|
41
|
-
const scale = renderScale(doc.pageSize(page).width);
|
|
42
|
-
const jpeg = await rasterToModelJpeg(await doc.renderAsync(page, { scale }));
|
|
43
|
-
if (!jpeg) {
|
|
44
|
-
throw new Error('no JPEG encoder: bare-media did not load');
|
|
45
|
-
}
|
|
46
|
-
return jpeg;
|
|
47
|
-
},
|
|
48
|
-
close() {
|
|
49
|
-
try {
|
|
50
|
-
doc.close();
|
|
51
|
-
}
|
|
52
|
-
catch { }
|
|
53
|
-
}
|
|
54
|
-
};
|
|
104
|
+
return wrapDoc(doc);
|
|
55
105
|
}
|
|
56
106
|
});
|
package/dist-lib/lib/core.js
CHANGED
|
@@ -52,7 +52,7 @@ import { forwardToDevices, servedToSiblings } from './api/device-routes.js';
|
|
|
52
52
|
import { OtaConfigReader, configCoreFor, resolveConfigChannel, vendoredMinSupportedVersion } from './core/ota-config.js';
|
|
53
53
|
import { updateRequired } from './core/min-version.js';
|
|
54
54
|
import { PendingDepartures } from './core/departures.js';
|
|
55
|
-
import { Mesh } from './core/mesh.js';
|
|
55
|
+
import { Mesh, SCHEMA_VERSION } from './core/mesh.js';
|
|
56
56
|
import { raceResource } from './race-resource.js';
|
|
57
57
|
import { createSystemInfoSampler } from './core/system-info.js';
|
|
58
58
|
import { settleOnClose } from '@qvac/harness/rpc';
|
|
@@ -345,6 +345,12 @@ export class Core extends ReadyResource {
|
|
|
345
345
|
return [...new Set(ids)].sort();
|
|
346
346
|
}
|
|
347
347
|
async _open() {
|
|
348
|
+
this._log?.info('lifecycle', 'core opening', {
|
|
349
|
+
app: this._app?.version ?? null,
|
|
350
|
+
build: this._app?.build ?? null,
|
|
351
|
+
channel: this._app?.channel ?? null,
|
|
352
|
+
schema: SCHEMA_VERSION
|
|
353
|
+
});
|
|
348
354
|
this._runBootOffline().catch(noop);
|
|
349
355
|
this._runBootOnline().catch(noop);
|
|
350
356
|
await this._offlineBoot.ready();
|
|
@@ -2611,9 +2617,15 @@ export class Core extends ReadyResource {
|
|
|
2611
2617
|
if (await this.local.files.backfilled(mesh.key)) {
|
|
2612
2618
|
return;
|
|
2613
2619
|
}
|
|
2614
|
-
|
|
2620
|
+
const started = Date.now();
|
|
2621
|
+
const chunks = await backfillFileIndex(mesh);
|
|
2615
2622
|
if (!mesh.closing) {
|
|
2616
2623
|
await this.local.files.markBackfilled(mesh.key);
|
|
2624
|
+
this._log?.info(['files'], 'recorded the files the mesh already held', {
|
|
2625
|
+
mesh: b4a.toHex(mesh.key).slice(0, 8),
|
|
2626
|
+
chunks,
|
|
2627
|
+
ms: Date.now() - started
|
|
2628
|
+
});
|
|
2617
2629
|
}
|
|
2618
2630
|
}
|
|
2619
2631
|
catch (err) {
|
|
@@ -2629,7 +2641,13 @@ export class Core extends ReadyResource {
|
|
|
2629
2641
|
try {
|
|
2630
2642
|
await mesh.waitForWritable();
|
|
2631
2643
|
await mesh.autobee.update();
|
|
2632
|
-
|
|
2644
|
+
const started = Date.now();
|
|
2645
|
+
const copied = await backfillChatChunks(mesh);
|
|
2646
|
+
this._log?.info(['p2p', 'mesh'], 'copied chats into chat-chunks', {
|
|
2647
|
+
mesh: b4a.toHex(mesh.key).slice(0, 8),
|
|
2648
|
+
copied,
|
|
2649
|
+
ms: Date.now() - started
|
|
2650
|
+
});
|
|
2633
2651
|
}
|
|
2634
2652
|
catch (err) {
|
|
2635
2653
|
if (!mesh.closing) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@qvac/core",
|
|
3
|
-
"version": "0.11.
|
|
3
|
+
"version": "0.11.4",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Peer-to-peer AI across your devices",
|
|
6
6
|
"engines": {
|
|
@@ -266,7 +266,7 @@
|
|
|
266
266
|
"bare-module": "~7.4.0",
|
|
267
267
|
"bare-os": "^3.9.1",
|
|
268
268
|
"bare-path": "^3.0.1",
|
|
269
|
-
"bare-pdfium": "^1.
|
|
269
|
+
"bare-pdfium": "^1.8.0",
|
|
270
270
|
"bare-pipe": "^4.2.2",
|
|
271
271
|
"bare-process": "^4.5.1",
|
|
272
272
|
"bare-rpc": "^1.3.7",
|