@lobu/runtime-device 24.6.0 → 24.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/component.json +1 -1
- package/dist/dependencies.bun.lock +25 -41
- package/npm-shrinkwrap.json +116 -240
- package/package.json +4 -2
- package/vendor/cli/dist/chrome-extension/manifest.json +5 -4
- package/vendor/cli/dist/chrome-extension/tools.js +2 -3
- package/vendor/cli/dist/commands/dev.js +28 -4
- package/vendor/cli/dist/connectors/imessage.ts +953 -0
- package/vendor/cli/dist/connectors/linkedin.ts +20 -377
- package/vendor/cli/dist/runtime-components.json +4 -4
- package/vendor/cli/dist/templates/AGENTS.md.tmpl +1 -1
- package/vendor/cli/package.json +1 -1
- package/vendor/connector-sdk/package.json +1 -1
- package/vendor/connector-worker/dist/env.d.ts +15 -0
- package/vendor/connector-worker/dist/env.js +17 -0
- package/vendor/connector-worker/dist/executor/isolate.d.ts +3 -1
- package/vendor/connector-worker/dist/executor/isolate.js +29 -8
- package/vendor/connector-worker/package.json +1 -1
- package/vendor/core/dist/connector-query-errors.d.ts +6 -0
- package/vendor/core/dist/connector-query-errors.js +4 -0
- package/vendor/core/dist/contracts/tools/manage-entity.d.ts +2 -0
- package/vendor/core/dist/contracts/tools/manage-entity.js +7 -0
- package/vendor/core/dist/contracts/tools/manage-feeds.d.ts +2 -0
- package/vendor/core/dist/contracts/tools/manage-feeds.js +3 -0
- package/vendor/core/dist/contracts/tools/view-host.js +2 -0
- package/vendor/core/dist/error-report.js +1 -0
- package/vendor/core/package.json +1 -1
- package/vendor/views/package.json +1 -1
- package/vendor/cli/dist/connectors/linkedin-takeout.ts +0 -577
- package/vendor/cli/dist/connectors/takeout-utils.ts +0 -240
|
@@ -1,240 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Pure takeout parsing/formatting helpers for the LinkedIn data-export feeds
|
|
3
|
-
* (`linkedin.ts`, `linkedin-takeout.ts`).
|
|
4
|
-
*
|
|
5
|
-
* ISOLATE-SAFE BY CONTRACT: this module must never import `node:fs`,
|
|
6
|
-
* `node:path`, or any other builtin the isolate prelude does not provide.
|
|
7
|
-
* Connector bundles run in a V8 isolate and `assertIsolateEligible` rejects the
|
|
8
|
-
* WHOLE bundle over one surviving builtin `require`, so an `fs` import here
|
|
9
|
-
* would take down the live browser feeds of every connector that shares these
|
|
10
|
-
* helpers — the LinkedIn regression in Lobu#3392.
|
|
11
|
-
*
|
|
12
|
-
* `node:crypto` is fine: the guest prelude serves it from `global.crypto`
|
|
13
|
-
* (see ISOLATE_PRELUDE_PROVIDED_BUILTINS).
|
|
14
|
-
*
|
|
15
|
-
* The CSV reader is injected into the connector (see its constructor), never
|
|
16
|
-
* imported, so no filesystem code reaches the bundle.
|
|
17
|
-
*/
|
|
18
|
-
|
|
19
|
-
import { createHash } from "node:crypto";
|
|
20
|
-
import type { EventEnvelope } from "@lobu/connector-sdk";
|
|
21
|
-
|
|
22
|
-
const DEFAULT_WATERMARK = "1970-01-01T00:00:00.000Z";
|
|
23
|
-
const DEFAULT_BATCH_SIZE = 1000;
|
|
24
|
-
|
|
25
|
-
export interface LocalTakeoutConfig {
|
|
26
|
-
takeout_dir?: string;
|
|
27
|
-
batch_size?: number;
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
export function batchSize(config: LocalTakeoutConfig): number {
|
|
31
|
-
return Math.max(1, Math.min(config.batch_size ?? DEFAULT_BATCH_SIZE, 5000));
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
export function takeBatch<T extends EventEnvelope>(
|
|
35
|
-
events: T[],
|
|
36
|
-
watermark: string | undefined,
|
|
37
|
-
max: number
|
|
38
|
-
): T[] {
|
|
39
|
-
const after = watermark ?? DEFAULT_WATERMARK;
|
|
40
|
-
return events
|
|
41
|
-
.filter((event) => eventCursor(event) > after)
|
|
42
|
-
.sort((a, b) => eventCursor(a).localeCompare(eventCursor(b)))
|
|
43
|
-
.slice(0, max);
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
export function maxEventCursor(
|
|
47
|
-
events: EventEnvelope[],
|
|
48
|
-
fallback?: string
|
|
49
|
-
): string {
|
|
50
|
-
return events.reduce(
|
|
51
|
-
(max, event) => (eventCursor(event) > max ? eventCursor(event) : max),
|
|
52
|
-
fallback ?? DEFAULT_WATERMARK
|
|
53
|
-
);
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
// Composite watermark: ISO timestamp + origin_id so events sharing a second
|
|
57
|
-
// still order deterministically. The delimiter is persisted inside the jsonb
|
|
58
|
-
// checkpoint column, so it must NOT be NUL: Postgres rejects NUL in jsonb
|
|
59
|
-
// (unsupported Unicode escape sequence). \u0001 (SOH) sorts before every
|
|
60
|
-
// printable char (preserving timestamp-first ordering) and is jsonb-safe.
|
|
61
|
-
const CURSOR_DELIMITER = "\u0001";
|
|
62
|
-
|
|
63
|
-
function eventCursor(event: EventEnvelope): string {
|
|
64
|
-
return `${event.occurred_at.toISOString()}${CURSOR_DELIMITER}${event.origin_id}`;
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
export function stableId(
|
|
68
|
-
prefix: string,
|
|
69
|
-
parts: Array<string | number | undefined>
|
|
70
|
-
): string {
|
|
71
|
-
const hash = createHash("sha256")
|
|
72
|
-
.update(parts.map((part) => String(part ?? "")).join("\0"))
|
|
73
|
-
.digest("hex")
|
|
74
|
-
.slice(0, 32);
|
|
75
|
-
return `${prefix}_${hash}`;
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
export function stripHtml(html: string): string {
|
|
79
|
-
return decodeHtml(htmlToPlainText(html))
|
|
80
|
-
.replace(/\n{3,}/g, "\n\n")
|
|
81
|
-
.replace(/[ \t]{2,}/g, " ")
|
|
82
|
-
.trim();
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
function htmlToPlainText(html: string): string {
|
|
86
|
-
let output = "";
|
|
87
|
-
let index = 0;
|
|
88
|
-
const lowerHtml = html.toLowerCase();
|
|
89
|
-
|
|
90
|
-
while (index < html.length) {
|
|
91
|
-
const char = html[index];
|
|
92
|
-
if (char !== "<") {
|
|
93
|
-
output += char;
|
|
94
|
-
index += 1;
|
|
95
|
-
continue;
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
const tagEnd = html.indexOf(">", index + 1);
|
|
99
|
-
if (tagEnd === -1) {
|
|
100
|
-
output += char;
|
|
101
|
-
index += 1;
|
|
102
|
-
continue;
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
const tagName = readTagName(html.slice(index + 1, tagEnd));
|
|
106
|
-
if (tagName === "script" || tagName === "style") {
|
|
107
|
-
const closeTagStart = lowerHtml.indexOf(`</${tagName}`, tagEnd + 1);
|
|
108
|
-
if (closeTagStart === -1) {
|
|
109
|
-
break;
|
|
110
|
-
}
|
|
111
|
-
const closeTagEnd = html.indexOf(">", closeTagStart + tagName.length + 2);
|
|
112
|
-
index = closeTagEnd === -1 ? html.length : closeTagEnd + 1;
|
|
113
|
-
continue;
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
if (tagName === "br" || isBlockBreakTag(tagName)) {
|
|
117
|
-
output += "\n";
|
|
118
|
-
}
|
|
119
|
-
index = tagEnd + 1;
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
return output;
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
function readTagName(tag: string): string {
|
|
126
|
-
let index = tag[0] === "/" ? 1 : 0;
|
|
127
|
-
while (tag[index] === " " || tag[index] === "\t" || tag[index] === "\n") {
|
|
128
|
-
index += 1;
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
let name = "";
|
|
132
|
-
while (index < tag.length) {
|
|
133
|
-
const char = tag[index]?.toLowerCase();
|
|
134
|
-
if (!char || !isAsciiNameChar(char)) {
|
|
135
|
-
break;
|
|
136
|
-
}
|
|
137
|
-
name += char;
|
|
138
|
-
index += 1;
|
|
139
|
-
}
|
|
140
|
-
return name;
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
function isAsciiNameChar(char: string): boolean {
|
|
144
|
-
return (
|
|
145
|
-
(char >= "a" && char <= "z") || (char >= "0" && char <= "9") || char === "-"
|
|
146
|
-
);
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
function isBlockBreakTag(tagName: string): boolean {
|
|
150
|
-
return (
|
|
151
|
-
tagName === "p" ||
|
|
152
|
-
tagName === "div" ||
|
|
153
|
-
tagName === "li" ||
|
|
154
|
-
tagName === "h1" ||
|
|
155
|
-
tagName === "h2" ||
|
|
156
|
-
tagName === "h3" ||
|
|
157
|
-
tagName === "h4" ||
|
|
158
|
-
tagName === "h5" ||
|
|
159
|
-
tagName === "h6"
|
|
160
|
-
);
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
export function decodeHtml(text: string): string {
|
|
164
|
-
return text.replace(
|
|
165
|
-
/&(nbsp|amp|quot|#039|apos|#\d+|#x[0-9a-f]+);/gi,
|
|
166
|
-
(entity, code) => {
|
|
167
|
-
const normalized = String(code).toLowerCase();
|
|
168
|
-
if (normalized === "nbsp") return " ";
|
|
169
|
-
if (normalized === "amp") return "&";
|
|
170
|
-
if (normalized === "quot") return '"';
|
|
171
|
-
if (normalized === "#039" || normalized === "apos") return "'";
|
|
172
|
-
if (normalized.startsWith("#x")) {
|
|
173
|
-
const codePoint = Number.parseInt(normalized.slice(2), 16);
|
|
174
|
-
return decodeHtmlCodePoint(codePoint, entity);
|
|
175
|
-
}
|
|
176
|
-
if (normalized.startsWith("#")) {
|
|
177
|
-
const codePoint = Number(normalized.slice(1));
|
|
178
|
-
return decodeHtmlCodePoint(codePoint, entity);
|
|
179
|
-
}
|
|
180
|
-
return entity;
|
|
181
|
-
}
|
|
182
|
-
);
|
|
183
|
-
}
|
|
184
|
-
|
|
185
|
-
function decodeHtmlCodePoint(codePoint: number, fallback: string): string {
|
|
186
|
-
if (!Number.isInteger(codePoint) || codePoint < 0 || codePoint > 0x10ffff) {
|
|
187
|
-
return fallback;
|
|
188
|
-
}
|
|
189
|
-
if (codePoint === 60 || codePoint === 62) {
|
|
190
|
-
return fallback;
|
|
191
|
-
}
|
|
192
|
-
return String.fromCodePoint(codePoint);
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
export function parseCsv(text: string): Record<string, string>[] {
|
|
196
|
-
const rows: string[][] = [];
|
|
197
|
-
let row: string[] = [];
|
|
198
|
-
let cell = "";
|
|
199
|
-
let inQuotes = false;
|
|
200
|
-
|
|
201
|
-
for (let i = 0; i < text.length; i++) {
|
|
202
|
-
const char = text[i];
|
|
203
|
-
const next = text[i + 1];
|
|
204
|
-
if (char === '"') {
|
|
205
|
-
if (inQuotes && next === '"') {
|
|
206
|
-
cell += '"';
|
|
207
|
-
i++;
|
|
208
|
-
} else {
|
|
209
|
-
inQuotes = !inQuotes;
|
|
210
|
-
}
|
|
211
|
-
} else if (char === "," && !inQuotes) {
|
|
212
|
-
row.push(cell);
|
|
213
|
-
cell = "";
|
|
214
|
-
} else if ((char === "\n" || char === "\r") && !inQuotes) {
|
|
215
|
-
if (char === "\r" && next === "\n") i++;
|
|
216
|
-
row.push(cell);
|
|
217
|
-
if (row.some((value) => value.trim().length > 0)) rows.push(row);
|
|
218
|
-
row = [];
|
|
219
|
-
cell = "";
|
|
220
|
-
} else {
|
|
221
|
-
cell += char;
|
|
222
|
-
}
|
|
223
|
-
}
|
|
224
|
-
row.push(cell);
|
|
225
|
-
if (row.some((value) => value.trim().length > 0)) rows.push(row);
|
|
226
|
-
|
|
227
|
-
const headerIndex = rows.findIndex((candidate) => {
|
|
228
|
-
const values = candidate.map((value) => value.trim()).filter(Boolean);
|
|
229
|
-
return values.length > 1 && !values[0]?.startsWith("Notes:");
|
|
230
|
-
});
|
|
231
|
-
if (headerIndex < 0) return [];
|
|
232
|
-
const headers = rows[headerIndex].map((value) => value.trim());
|
|
233
|
-
return rows.slice(headerIndex + 1).map((values) => {
|
|
234
|
-
const record: Record<string, string> = {};
|
|
235
|
-
headers.forEach((header, index) => {
|
|
236
|
-
record[header] = values[index]?.trim() ?? "";
|
|
237
|
-
});
|
|
238
|
-
return record;
|
|
239
|
-
});
|
|
240
|
-
}
|