@hraness/message-like-me 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/README.md +139 -54
- package/dist/cli.js +1 -1
- package/docs/local-message-bundle-v1.md +22 -9
- package/docs/local-message-bundle-v2.md +24 -9
- package/docs/publishing.md +633 -0
- package/package.json +7 -1
- package/skills/ensoul/SKILL.md +4 -4
- package/skills/ensoul/VENDORED_FROM.md +1 -1
- package/skills/ensoul/agents/openai.yaml +1 -1
- package/skills/ensoul/references/source-packets.md +1 -1
- package/skills/ensoul/scripts/prepare-x-archive.ts +411 -0
- package/skills/ensoul/scripts/source-packet.ts +372 -0
- package/skills/ensoul/scripts/validate-source-packet.ts +638 -0
- package/skills/ensoul/scripts/x-zip-file.ts +419 -0
- package/skills/message-like-me/SKILL.md +8 -1
- package/skills/message-like-me/references/privacy.md +3 -1
- package/skills/ensoul/scripts/prepare_x_archive.py +0 -467
- package/skills/ensoul/scripts/validate_source_packet.py +0 -477
|
@@ -0,0 +1,419 @@
|
|
|
1
|
+
import { readSync } from "node:fs";
|
|
2
|
+
import { inflateRawSync } from "node:zlib";
|
|
3
|
+
|
|
4
|
+
export type ExtractedZipMember = Readonly<{ memberName: string; bytes: Uint8Array }>;
|
|
5
|
+
|
|
6
|
+
const EOCD = 0x06054b50;
|
|
7
|
+
const ZIP64_EOCD = 0x06064b50;
|
|
8
|
+
const ZIP64_LOCATOR = 0x07064b50;
|
|
9
|
+
const CENTRAL = 0x02014b50;
|
|
10
|
+
const LOCAL = 0x04034b50;
|
|
11
|
+
const DESCRIPTOR = 0x08074b50;
|
|
12
|
+
const ZIP64_EXTRA = 0x0001;
|
|
13
|
+
const STORED = 0;
|
|
14
|
+
const DEFLATE = 8;
|
|
15
|
+
const DESCRIPTOR_FLAG = 0x0008;
|
|
16
|
+
const UTF8_FLAG = 0x0800;
|
|
17
|
+
const ENCRYPTED_FLAG = 0x0001;
|
|
18
|
+
const STRONG_ENCRYPTION_FLAG = 0x0040;
|
|
19
|
+
const MASKED_HEADER_FLAG = 0x2000;
|
|
20
|
+
const DEFLATE_OPTION_FLAGS = 0x0006;
|
|
21
|
+
const UNIX_HOST = 3;
|
|
22
|
+
const MACOS_HOST = 19;
|
|
23
|
+
const UNIX_TYPE_MASK = 0o170000;
|
|
24
|
+
const UNIX_REGULAR = 0o100000;
|
|
25
|
+
const UNIX_DIRECTORY = 0o040000;
|
|
26
|
+
const DOS_DIRECTORY = 0x10;
|
|
27
|
+
const U16_MAX = 0xffff;
|
|
28
|
+
const U32_MAX = 0xffffffff;
|
|
29
|
+
|
|
30
|
+
export const MAX_X_ZIP_ARCHIVE_BYTES = 16 * 1024 * 1024 * 1024;
|
|
31
|
+
export const MAX_X_ZIP_MEMBER_BYTES = 256 * 1024 * 1024;
|
|
32
|
+
const MAX_X_COMPRESSED_MEMBER_BYTES = MAX_X_ZIP_MEMBER_BYTES;
|
|
33
|
+
const MAX_X_ENTRIES = 100_000;
|
|
34
|
+
const MAX_X_CENTRAL_BYTES = 64 * 1024 * 1024;
|
|
35
|
+
const MAX_X_TOTAL_SELECTED_BYTES = 512 * 1024 * 1024;
|
|
36
|
+
const MAX_X_TOTAL_DECLARED_BYTES = 64 * 1024 * 1024 * 1024;
|
|
37
|
+
const MAX_NAME_BYTES = 4 * 1024;
|
|
38
|
+
const MAX_RATIO = 1_000;
|
|
39
|
+
|
|
40
|
+
type Directory = Readonly<{ count: number; offset: number; end: number }>;
|
|
41
|
+
type Entry = Readonly<{
|
|
42
|
+
name: string;
|
|
43
|
+
nameBytes: Buffer;
|
|
44
|
+
directory: boolean;
|
|
45
|
+
selected: boolean;
|
|
46
|
+
versionMadeBy: number;
|
|
47
|
+
versionNeeded: number;
|
|
48
|
+
flags: number;
|
|
49
|
+
method: number;
|
|
50
|
+
modifiedTime: number;
|
|
51
|
+
modifiedDate: number;
|
|
52
|
+
crc32: number;
|
|
53
|
+
compressedSize: number;
|
|
54
|
+
uncompressedSize: number;
|
|
55
|
+
externalAttributes: number;
|
|
56
|
+
localHeaderOffset: number;
|
|
57
|
+
}>;
|
|
58
|
+
type LocalRange = Readonly<{ dataOffset: number; dataEnd: number }>;
|
|
59
|
+
|
|
60
|
+
const X_POSTS = /(?:^|\/)data\/tweets(?:-part\d+)?\.js$/u;
|
|
61
|
+
|
|
62
|
+
const CRC32_TABLE = (() => {
|
|
63
|
+
const table = new Uint32Array(256);
|
|
64
|
+
for (let index = 0; index < table.length; index += 1) {
|
|
65
|
+
let value = index;
|
|
66
|
+
for (let bit = 0; bit < 8; bit += 1) value = (value & 1) === 1 ? 0xedb88320 ^ (value >>> 1) : value >>> 1;
|
|
67
|
+
table[index] = value >>> 0;
|
|
68
|
+
}
|
|
69
|
+
return table;
|
|
70
|
+
})();
|
|
71
|
+
|
|
72
|
+
function updateCrc32(state: number, bytes: Uint8Array): number {
|
|
73
|
+
let value = state;
|
|
74
|
+
for (const byte of bytes) value = CRC32_TABLE[(value ^ byte) & 0xff]! ^ (value >>> 8);
|
|
75
|
+
return value;
|
|
76
|
+
}
|
|
77
|
+
function checkedEnd(offset: number, length: number, label: string): number {
|
|
78
|
+
const value = offset + length;
|
|
79
|
+
if (!Number.isSafeInteger(offset) || !Number.isSafeInteger(length) || offset < 0 || length < 0 || !Number.isSafeInteger(value) || value < offset) {
|
|
80
|
+
throw new Error(`X ZIP ${label} has invalid bounds`);
|
|
81
|
+
}
|
|
82
|
+
return value;
|
|
83
|
+
}
|
|
84
|
+
function readExact(descriptor: number, offset: number, length: number, label: string): Buffer {
|
|
85
|
+
checkedEnd(offset, length, label);
|
|
86
|
+
const bytes = Buffer.allocUnsafe(length);
|
|
87
|
+
let position = 0;
|
|
88
|
+
while (position < length) {
|
|
89
|
+
const count = readSync(descriptor, bytes, position, length - position, offset + position);
|
|
90
|
+
if (count < 1) throw new Error(`X ZIP ${label} is truncated`);
|
|
91
|
+
position += count;
|
|
92
|
+
}
|
|
93
|
+
return bytes;
|
|
94
|
+
}
|
|
95
|
+
function u64(bytes: Buffer, offset: number, label: string): number {
|
|
96
|
+
if (offset < 0 || bytes.length - offset < 8) throw new Error(`X ZIP ${label} is truncated`);
|
|
97
|
+
const value = bytes.readBigUInt64LE(offset);
|
|
98
|
+
if (value > BigInt(Number.MAX_SAFE_INTEGER)) throw new Error(`X ZIP ${label} exceeds JavaScript's exact integer range`);
|
|
99
|
+
return Number(value);
|
|
100
|
+
}
|
|
101
|
+
function fieldValue(legacy: number, resolved: number, sentinel: number, label: string): void {
|
|
102
|
+
if (legacy !== sentinel && legacy !== resolved) throw new Error(`X ZIP legacy ${label} contradicts ZIP64 metadata`);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function directoryFromEocd(descriptor: number, archiveSize: number): Directory {
|
|
106
|
+
const tailLength = Math.min(archiveSize, 22 + U16_MAX + 20);
|
|
107
|
+
const tailOffset = archiveSize - tailLength;
|
|
108
|
+
const tail = readExact(descriptor, tailOffset, tailLength, "end records");
|
|
109
|
+
const candidates: number[] = [];
|
|
110
|
+
for (let offset = tail.length - 22; offset >= 0; offset -= 1) {
|
|
111
|
+
if (tail.readUInt32LE(offset) === EOCD && tailOffset + offset + 22 + tail.readUInt16LE(offset + 20) === archiveSize) {
|
|
112
|
+
candidates.push(tailOffset + offset);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
if (candidates.length !== 1) {
|
|
116
|
+
throw new Error(candidates.length === 0 ? "X ZIP end-of-central-directory record is missing" : "X ZIP end-of-central-directory record is ambiguous");
|
|
117
|
+
}
|
|
118
|
+
const eocdOffset = candidates[0]!;
|
|
119
|
+
const eocd = readExact(descriptor, eocdOffset, archiveSize - eocdOffset, "end-of-central-directory record");
|
|
120
|
+
if (eocd.length !== 22 || eocd.readUInt16LE(20) !== 0) throw new Error("X ZIP archive comments are not supported");
|
|
121
|
+
const legacyDisk = eocd.readUInt16LE(4);
|
|
122
|
+
const legacyCentralDisk = eocd.readUInt16LE(6);
|
|
123
|
+
const legacyOnDisk = eocd.readUInt16LE(8);
|
|
124
|
+
const legacyCount = eocd.readUInt16LE(10);
|
|
125
|
+
const legacySize = eocd.readUInt32LE(12);
|
|
126
|
+
const legacyOffset = eocd.readUInt32LE(16);
|
|
127
|
+
const locatorOffset = eocdOffset - 20;
|
|
128
|
+
const hasZip64 = locatorOffset >= 0 && readExact(descriptor, locatorOffset, 4, "ZIP64 locator signature").readUInt32LE(0) === ZIP64_LOCATOR;
|
|
129
|
+
if (!hasZip64) {
|
|
130
|
+
if ([legacyDisk, legacyCentralDisk, legacyOnDisk, legacyCount].includes(U16_MAX)
|
|
131
|
+
|| [legacySize, legacyOffset].includes(U32_MAX)) throw new Error("X ZIP archive is missing required ZIP64 end metadata");
|
|
132
|
+
if (legacyDisk !== 0 || legacyCentralDisk !== 0 || legacyOnDisk !== legacyCount) throw new Error("X ZIP multi-disk archives are not supported");
|
|
133
|
+
if (legacyCount < 1 || legacyCount > MAX_X_ENTRIES || legacySize > MAX_X_CENTRAL_BYTES) throw new Error("X ZIP central directory exceeds its bounds");
|
|
134
|
+
const end = checkedEnd(legacyOffset, legacySize, "central directory");
|
|
135
|
+
if (end !== eocdOffset) throw new Error("X ZIP central directory has invalid bounds");
|
|
136
|
+
return { count: legacyCount, offset: legacyOffset, end };
|
|
137
|
+
}
|
|
138
|
+
const locator = readExact(descriptor, locatorOffset, 20, "ZIP64 locator");
|
|
139
|
+
if (locator.readUInt32LE(4) !== 0 || locator.readUInt32LE(16) !== 1) throw new Error("X ZIP multi-disk archives are not supported");
|
|
140
|
+
const zip64Offset = u64(locator, 8, "ZIP64 end record offset");
|
|
141
|
+
const zip64 = readExact(descriptor, zip64Offset, 56, "ZIP64 end record");
|
|
142
|
+
if (zip64.readUInt32LE(0) !== ZIP64_EOCD || u64(zip64, 4, "ZIP64 end record size") !== 44) {
|
|
143
|
+
throw new Error("X ZIP64 end record has an unsupported shape");
|
|
144
|
+
}
|
|
145
|
+
if (zip64Offset + zip64.length !== locatorOffset || zip64.readUInt32LE(16) !== 0 || zip64.readUInt32LE(20) !== 0) {
|
|
146
|
+
throw new Error("X ZIP64 end record has invalid bounds or disk ownership");
|
|
147
|
+
}
|
|
148
|
+
const onDisk = u64(zip64, 24, "ZIP64 entries on disk");
|
|
149
|
+
const count = u64(zip64, 32, "ZIP64 entry count");
|
|
150
|
+
const size = u64(zip64, 40, "ZIP64 central directory size");
|
|
151
|
+
const offset = u64(zip64, 48, "ZIP64 central directory offset");
|
|
152
|
+
if (onDisk !== count) throw new Error("X ZIP multi-disk archives are not supported");
|
|
153
|
+
if (count < 1 || count > MAX_X_ENTRIES || size > MAX_X_CENTRAL_BYTES) throw new Error("X ZIP central directory exceeds its bounds");
|
|
154
|
+
const end = checkedEnd(offset, size, "ZIP64 central directory");
|
|
155
|
+
if (end !== zip64Offset) throw new Error("X ZIP64 central directory has invalid bounds");
|
|
156
|
+
fieldValue(legacyDisk, 0, U16_MAX, "disk number");
|
|
157
|
+
fieldValue(legacyCentralDisk, 0, U16_MAX, "central disk number");
|
|
158
|
+
fieldValue(legacyOnDisk, onDisk, U16_MAX, "entries-on-disk count");
|
|
159
|
+
fieldValue(legacyCount, count, U16_MAX, "entry count");
|
|
160
|
+
fieldValue(legacySize, size, U32_MAX, "central directory size");
|
|
161
|
+
fieldValue(legacyOffset, offset, U32_MAX, "central directory offset");
|
|
162
|
+
return { count, offset, end };
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function extraFields(bytes: Buffer, label: string): Map<number, Buffer> {
|
|
166
|
+
const fields = new Map<number, Buffer>();
|
|
167
|
+
let position = 0;
|
|
168
|
+
while (position < bytes.length) {
|
|
169
|
+
if (bytes.length - position < 4) throw new Error(`X ZIP ${label} contains a truncated extra field`);
|
|
170
|
+
const id = bytes.readUInt16LE(position);
|
|
171
|
+
const length = bytes.readUInt16LE(position + 2);
|
|
172
|
+
const next = checkedEnd(position + 4, length, `${label} extra field`);
|
|
173
|
+
if (next > bytes.length) throw new Error(`X ZIP ${label} contains a truncated extra field`);
|
|
174
|
+
if (fields.has(id)) throw new Error(`X ZIP ${label} contains duplicate extra field ${id}`);
|
|
175
|
+
fields.set(id, bytes.subarray(position + 4, next));
|
|
176
|
+
position = next;
|
|
177
|
+
}
|
|
178
|
+
return fields;
|
|
179
|
+
}
|
|
180
|
+
function decodeName(nameBytes: Buffer, flags: number, selected: boolean): { name: string; directory: boolean } {
|
|
181
|
+
if (nameBytes.length < 1 || nameBytes.length > MAX_NAME_BYTES) throw new Error("X ZIP member name exceeds its bounds");
|
|
182
|
+
let decoded: string;
|
|
183
|
+
if ((flags & UTF8_FLAG) !== 0) {
|
|
184
|
+
try { decoded = new TextDecoder("utf-8", { fatal: true }).decode(nameBytes); }
|
|
185
|
+
catch (error) { throw new Error("X ZIP member name is not valid UTF-8", { cause: error }); }
|
|
186
|
+
} else {
|
|
187
|
+
if (nameBytes.some((byte) => byte > 0x7f)) throw new Error("X ZIP non-UTF-8 member names must be ASCII");
|
|
188
|
+
decoded = nameBytes.toString("ascii");
|
|
189
|
+
}
|
|
190
|
+
if (decoded.normalize("NFC") !== decoded) throw new Error("X ZIP member name is not NFC-normalized");
|
|
191
|
+
if (/^[A-Za-z]:\//u.test(decoded) || decoded.startsWith("/")) throw new Error("X ZIP member has an absolute name");
|
|
192
|
+
if (decoded.includes("\\")) throw new Error("X ZIP member name contains a backslash");
|
|
193
|
+
if (/[\u0000-\u001F\u007F-\u009F]/u.test(decoded)) throw new Error("X ZIP member name contains a control character");
|
|
194
|
+
const directory = decoded.endsWith("/");
|
|
195
|
+
const path = directory ? decoded.slice(0, -1) : decoded;
|
|
196
|
+
const parts = path.split("/");
|
|
197
|
+
if (parts.some((part) => part === "." || part === "..") || (selected && parts.some((part) => part === ""))) {
|
|
198
|
+
throw new Error("X ZIP selected member has an unsafe path component");
|
|
199
|
+
}
|
|
200
|
+
return { name: decoded, directory };
|
|
201
|
+
}
|
|
202
|
+
function validateFlags(flags: number, method: number): void {
|
|
203
|
+
if ((flags & (ENCRYPTED_FLAG | STRONG_ENCRYPTION_FLAG)) !== 0) throw new Error("X ZIP encrypted members are not supported");
|
|
204
|
+
if ((flags & MASKED_HEADER_FLAG) !== 0) throw new Error("X ZIP members with masked local headers are not supported");
|
|
205
|
+
const allowed = UTF8_FLAG | DESCRIPTOR_FLAG | (method === DEFLATE ? DEFLATE_OPTION_FLAGS : 0);
|
|
206
|
+
if ((flags & ~allowed) !== 0) throw new Error("X ZIP member uses unsupported general-purpose flags");
|
|
207
|
+
}
|
|
208
|
+
function validateType(entry: Entry): void {
|
|
209
|
+
const host = entry.versionMadeBy >>> 8;
|
|
210
|
+
const unixType = host === UNIX_HOST || host === MACOS_HOST ? (entry.externalAttributes >>> 16) & UNIX_TYPE_MASK : 0;
|
|
211
|
+
if (entry.directory) {
|
|
212
|
+
if (entry.crc32 !== 0 || entry.uncompressedSize !== 0 || (entry.method === STORED && entry.compressedSize !== 0)) {
|
|
213
|
+
throw new Error("X ZIP directory member must expand to empty data");
|
|
214
|
+
}
|
|
215
|
+
if (unixType !== 0 && unixType !== UNIX_DIRECTORY) throw new Error("X ZIP member is a symlink or another non-regular file");
|
|
216
|
+
} else if ((entry.externalAttributes & DOS_DIRECTORY) !== 0 || (unixType !== 0 && unixType !== UNIX_REGULAR)) {
|
|
217
|
+
throw new Error("X ZIP member is not a regular file");
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
function selectedName(name: string): boolean {
|
|
221
|
+
return X_POSTS.test(name);
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
function centralEntries(descriptor: number, directory: Directory, archiveSize: number): Entry[] {
|
|
225
|
+
const bytes = readExact(descriptor, directory.offset, directory.end - directory.offset, "central directory");
|
|
226
|
+
const entries: Entry[] = [];
|
|
227
|
+
const names = new Set<string>();
|
|
228
|
+
let declared = 0;
|
|
229
|
+
let selectedTotal = 0;
|
|
230
|
+
let position = 0;
|
|
231
|
+
for (let index = 0; index < directory.count; index += 1) {
|
|
232
|
+
if (bytes.length - position < 46 || bytes.readUInt32LE(position) !== CENTRAL) throw new Error("X ZIP central directory entry is invalid or truncated");
|
|
233
|
+
const versionMadeBy = bytes.readUInt16LE(position + 4);
|
|
234
|
+
const versionNeeded = bytes.readUInt16LE(position + 6);
|
|
235
|
+
const flags = bytes.readUInt16LE(position + 8);
|
|
236
|
+
const method = bytes.readUInt16LE(position + 10);
|
|
237
|
+
const modifiedTime = bytes.readUInt16LE(position + 12);
|
|
238
|
+
const modifiedDate = bytes.readUInt16LE(position + 14);
|
|
239
|
+
const checksum = bytes.readUInt32LE(position + 16);
|
|
240
|
+
const compressedLegacy = bytes.readUInt32LE(position + 20);
|
|
241
|
+
const uncompressedLegacy = bytes.readUInt32LE(position + 24);
|
|
242
|
+
const nameLength = bytes.readUInt16LE(position + 28);
|
|
243
|
+
const extraLength = bytes.readUInt16LE(position + 30);
|
|
244
|
+
const commentLength = bytes.readUInt16LE(position + 32);
|
|
245
|
+
const diskLegacy = bytes.readUInt16LE(position + 34);
|
|
246
|
+
const externalAttributes = bytes.readUInt32LE(position + 38);
|
|
247
|
+
const offsetLegacy = bytes.readUInt32LE(position + 42);
|
|
248
|
+
const next = checkedEnd(position + 46, nameLength + extraLength + commentLength, "central directory entry");
|
|
249
|
+
if (next > bytes.length) throw new Error("X ZIP central directory entry is truncated");
|
|
250
|
+
const nameBytes = Buffer.from(bytes.subarray(position + 46, position + 46 + nameLength));
|
|
251
|
+
const extraStart = position + 46 + nameLength;
|
|
252
|
+
const fields = extraFields(bytes.subarray(extraStart, extraStart + extraLength), `central directory entry ${index + 1}`);
|
|
253
|
+
const zip64 = fields.get(ZIP64_EXTRA) ?? null;
|
|
254
|
+
let zip64Position = 0;
|
|
255
|
+
const nextZip64 = (label: string) => {
|
|
256
|
+
if (zip64 === null) throw new Error(`X ZIP ${label} is missing ZIP64 metadata`);
|
|
257
|
+
const value = u64(zip64, zip64Position, label); zip64Position += 8; return value;
|
|
258
|
+
};
|
|
259
|
+
const uncompressedSize = uncompressedLegacy === U32_MAX ? nextZip64("uncompressed size") : uncompressedLegacy;
|
|
260
|
+
const compressedSize = compressedLegacy === U32_MAX ? nextZip64("compressed size") : compressedLegacy;
|
|
261
|
+
const localHeaderOffset = offsetLegacy === U32_MAX ? nextZip64("local header offset") : offsetLegacy;
|
|
262
|
+
let disk = diskLegacy;
|
|
263
|
+
if (diskLegacy === U16_MAX) {
|
|
264
|
+
if (zip64 === null || zip64.length - zip64Position < 4) throw new Error("X ZIP disk number is missing ZIP64 metadata");
|
|
265
|
+
disk = zip64.readUInt32LE(zip64Position); zip64Position += 4;
|
|
266
|
+
}
|
|
267
|
+
if ((zip64 === null && zip64Position !== 0) || (zip64 !== null && zip64Position !== zip64.length)) {
|
|
268
|
+
throw new Error("X ZIP central directory contains ambiguous ZIP64 metadata");
|
|
269
|
+
}
|
|
270
|
+
if (disk !== 0) throw new Error("X ZIP multi-disk archives are not supported");
|
|
271
|
+
validateFlags(flags, method);
|
|
272
|
+
if (method !== STORED && method !== DEFLATE) throw new Error(`X ZIP compression method ${method} is unsupported`);
|
|
273
|
+
const provisionalName = decodeName(nameBytes, flags, false);
|
|
274
|
+
const selected = !provisionalName.directory && selectedName(provisionalName.name);
|
|
275
|
+
const decoded = selected ? decodeName(nameBytes, flags, true) : provisionalName;
|
|
276
|
+
const entry: Entry = { name: decoded.name, nameBytes, directory: decoded.directory, selected, versionMadeBy, versionNeeded,
|
|
277
|
+
flags, method, modifiedTime, modifiedDate, crc32: checksum, compressedSize, uncompressedSize,
|
|
278
|
+
externalAttributes, localHeaderOffset };
|
|
279
|
+
if (names.has(entry.name)) throw new Error("X ZIP archive contains duplicate member names");
|
|
280
|
+
names.add(entry.name);
|
|
281
|
+
if (![compressedSize, uncompressedSize, localHeaderOffset].every((value) => Number.isSafeInteger(value) && value >= 0)) {
|
|
282
|
+
throw new Error("X ZIP member has an invalid size or offset");
|
|
283
|
+
}
|
|
284
|
+
if (compressedSize > archiveSize || localHeaderOffset >= directory.offset) throw new Error("X ZIP member exceeds archive bounds");
|
|
285
|
+
if (method === STORED && compressedSize !== uncompressedSize) throw new Error("X ZIP stored member has inconsistent sizes");
|
|
286
|
+
if (selected && (compressedSize < 1 || uncompressedSize < 1 || compressedSize > MAX_X_COMPRESSED_MEMBER_BYTES || uncompressedSize > MAX_X_ZIP_MEMBER_BYTES)) {
|
|
287
|
+
throw new Error("selected X ZIP member exceeds its size bounds");
|
|
288
|
+
}
|
|
289
|
+
if (selected && uncompressedSize > compressedSize * MAX_RATIO) throw new Error("selected X ZIP member exceeds its compression-ratio limit");
|
|
290
|
+
declared += uncompressedSize;
|
|
291
|
+
if (!Number.isSafeInteger(declared) || declared > MAX_X_TOTAL_DECLARED_BYTES) throw new Error("X ZIP archive exceeds its declared uncompressed-size limit");
|
|
292
|
+
if (selected) {
|
|
293
|
+
selectedTotal += uncompressedSize;
|
|
294
|
+
if (!Number.isSafeInteger(selectedTotal) || selectedTotal > MAX_X_TOTAL_SELECTED_BYTES) throw new Error("selected X ZIP members exceed their total size limit");
|
|
295
|
+
}
|
|
296
|
+
validateType(entry);
|
|
297
|
+
entries.push(entry);
|
|
298
|
+
position = next;
|
|
299
|
+
}
|
|
300
|
+
if (position !== bytes.length) throw new Error("X ZIP central directory contains unindexed data");
|
|
301
|
+
return entries;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
function localSizes(compressed: number, uncompressed: number, fields: Map<number, Buffer>, label: string): { compressed: number; uncompressed: number; zip64: boolean } {
|
|
305
|
+
const zip64 = fields.get(ZIP64_EXTRA) ?? null;
|
|
306
|
+
if (compressed !== U32_MAX && uncompressed !== U32_MAX) {
|
|
307
|
+
if (zip64 !== null) throw new Error(`X ZIP ${label} contains redundant ZIP64 size metadata`);
|
|
308
|
+
return { compressed, uncompressed, zip64: false };
|
|
309
|
+
}
|
|
310
|
+
if (zip64 === null) throw new Error(`X ZIP ${label} is missing ZIP64 size metadata`);
|
|
311
|
+
let position = 0;
|
|
312
|
+
let resolvedUncompressed = uncompressed;
|
|
313
|
+
let resolvedCompressed = compressed;
|
|
314
|
+
if (uncompressed === U32_MAX) { resolvedUncompressed = u64(zip64, position, `${label} uncompressed size`); position += 8; }
|
|
315
|
+
if (compressed === U32_MAX) { resolvedCompressed = u64(zip64, position, `${label} compressed size`); position += 8; }
|
|
316
|
+
if (position !== zip64.length) throw new Error(`X ZIP ${label} contains ambiguous ZIP64 size metadata`);
|
|
317
|
+
return { compressed: resolvedCompressed, uncompressed: resolvedUncompressed, zip64: true };
|
|
318
|
+
}
|
|
319
|
+
function validateDescriptor(bytes: Buffer, entry: Entry): void {
|
|
320
|
+
let position = 0;
|
|
321
|
+
if (bytes.length === 16 || bytes.length === 24) {
|
|
322
|
+
if (bytes.readUInt32LE(0) !== DESCRIPTOR) throw new Error("X ZIP data descriptor signature is invalid");
|
|
323
|
+
position = 4;
|
|
324
|
+
} else if (bytes.length !== 12 && bytes.length !== 20) throw new Error("X ZIP data descriptor has an invalid length");
|
|
325
|
+
if (bytes.readUInt32LE(position) !== entry.crc32) throw new Error("X ZIP data descriptor checksum disagrees with the central directory");
|
|
326
|
+
position += 4;
|
|
327
|
+
const zip64 = bytes.length - position === 16;
|
|
328
|
+
const compressed = zip64 ? u64(bytes, position, "descriptor compressed size") : bytes.readUInt32LE(position);
|
|
329
|
+
position += zip64 ? 8 : 4;
|
|
330
|
+
const uncompressed = zip64 ? u64(bytes, position, "descriptor uncompressed size") : bytes.readUInt32LE(position);
|
|
331
|
+
if (compressed !== entry.compressedSize || uncompressed !== entry.uncompressedSize) throw new Error("X ZIP data descriptor sizes disagree with the central directory");
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
function localRanges(descriptor: number, entries: readonly Entry[], centralOffset: number): Map<number, LocalRange> {
|
|
335
|
+
const ordered = [...entries].sort((left, right) => left.localHeaderOffset - right.localHeaderOffset);
|
|
336
|
+
const ranges = new Map<number, LocalRange>();
|
|
337
|
+
let expected = 0;
|
|
338
|
+
for (const [index, entry] of ordered.entries()) {
|
|
339
|
+
const label = `local member ${index + 1}`;
|
|
340
|
+
const offset = entry.localHeaderOffset;
|
|
341
|
+
if (offset !== expected) throw new Error(offset < expected ? "X ZIP local member ranges overlap" : "X ZIP archive contains unindexed local data");
|
|
342
|
+
const header = readExact(descriptor, offset, 30, `${label} header`);
|
|
343
|
+
if (header.readUInt32LE(0) !== LOCAL) throw new Error("X ZIP local header signature is invalid");
|
|
344
|
+
const version = header.readUInt16LE(4);
|
|
345
|
+
const flags = header.readUInt16LE(6);
|
|
346
|
+
const method = header.readUInt16LE(8);
|
|
347
|
+
const modifiedTime = header.readUInt16LE(10);
|
|
348
|
+
const modifiedDate = header.readUInt16LE(12);
|
|
349
|
+
const checksum = header.readUInt32LE(14);
|
|
350
|
+
const compressedLegacy = header.readUInt32LE(18);
|
|
351
|
+
const uncompressedLegacy = header.readUInt32LE(22);
|
|
352
|
+
const nameLength = header.readUInt16LE(26);
|
|
353
|
+
const extraLength = header.readUInt16LE(28);
|
|
354
|
+
if (version !== entry.versionNeeded || flags !== entry.flags || method !== entry.method
|
|
355
|
+
|| modifiedTime !== entry.modifiedTime || modifiedDate !== entry.modifiedDate) throw new Error("X ZIP local header disagrees with the central directory");
|
|
356
|
+
const variable = readExact(descriptor, offset + 30, nameLength + extraLength, `${label} fields`);
|
|
357
|
+
if (!variable.subarray(0, nameLength).equals(entry.nameBytes)) throw new Error("X ZIP local member name disagrees with the central directory");
|
|
358
|
+
const fields = extraFields(variable.subarray(nameLength), `${label} header`);
|
|
359
|
+
const sizes = localSizes(compressedLegacy, uncompressedLegacy, fields, `${label} header`);
|
|
360
|
+
if ((flags & DESCRIPTOR_FLAG) === 0) {
|
|
361
|
+
if (checksum !== entry.crc32 || sizes.compressed !== entry.compressedSize || sizes.uncompressed !== entry.uncompressedSize) {
|
|
362
|
+
throw new Error("X ZIP local sizes or checksum disagree with the central directory");
|
|
363
|
+
}
|
|
364
|
+
} else if ((checksum !== 0 && checksum !== entry.crc32)
|
|
365
|
+
|| (sizes.compressed !== 0 && sizes.compressed !== entry.compressedSize)
|
|
366
|
+
|| (sizes.uncompressed !== 0 && sizes.uncompressed !== entry.uncompressedSize)) {
|
|
367
|
+
throw new Error("X ZIP local descriptor placeholders disagree with the central directory");
|
|
368
|
+
}
|
|
369
|
+
const dataOffset = checkedEnd(offset + 30, nameLength + extraLength, "local member data offset");
|
|
370
|
+
const dataEnd = checkedEnd(dataOffset, entry.compressedSize, "local member compressed data");
|
|
371
|
+
const next = index + 1 < ordered.length ? ordered[index + 1]!.localHeaderOffset : centralOffset;
|
|
372
|
+
if (dataEnd > next) throw new Error("X ZIP local member ranges overlap");
|
|
373
|
+
if ((flags & DESCRIPTOR_FLAG) === 0) {
|
|
374
|
+
if (dataEnd !== next) throw new Error("X ZIP archive contains unindexed local data");
|
|
375
|
+
} else {
|
|
376
|
+
const descriptorLength = next - dataEnd;
|
|
377
|
+
const allowedLengths = sizes.zip64 ? [20, 24] : [12, 16];
|
|
378
|
+
if (!allowedLengths.includes(descriptorLength)) throw new Error("X ZIP data descriptor has an invalid width");
|
|
379
|
+
validateDescriptor(readExact(descriptor, dataEnd, descriptorLength, `${label} descriptor`), entry);
|
|
380
|
+
}
|
|
381
|
+
ranges.set(offset, { dataOffset, dataEnd });
|
|
382
|
+
expected = next;
|
|
383
|
+
}
|
|
384
|
+
if (expected !== centralOffset) throw new Error("X ZIP archive contains unindexed local data");
|
|
385
|
+
return ranges;
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
function readSelected(descriptor: number, entry: Entry, range: LocalRange): ExtractedZipMember {
|
|
389
|
+
const compressed = readExact(descriptor, range.dataOffset, range.dataEnd - range.dataOffset, `selected member ${entry.name}`);
|
|
390
|
+
let output: Buffer;
|
|
391
|
+
try {
|
|
392
|
+
output = entry.method === STORED ? Buffer.from(compressed) : inflateRawSync(compressed, {
|
|
393
|
+
maxOutputLength: Math.min(MAX_X_ZIP_MEMBER_BYTES + 1, entry.uncompressedSize + 1),
|
|
394
|
+
});
|
|
395
|
+
} catch (error) {
|
|
396
|
+
throw new Error(`selected X ZIP member is invalid: ${entry.name}`, { cause: error });
|
|
397
|
+
}
|
|
398
|
+
if (output.length !== entry.uncompressedSize) throw new Error("selected X ZIP member has an incorrect output size");
|
|
399
|
+
if (((updateCrc32(0xffffffff, output) ^ 0xffffffff) >>> 0) !== entry.crc32) throw new Error("selected X ZIP member failed its CRC-32 check");
|
|
400
|
+
return { memberName: entry.name, bytes: output };
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
/** Strictly validate a large official X ZIP and read only authored-post JS members. */
|
|
404
|
+
export function extractXArchiveFile(descriptor: number, archiveSize: number): readonly ExtractedZipMember[] {
|
|
405
|
+
if (!Number.isSafeInteger(descriptor) || descriptor < 0) throw new Error("X ZIP descriptor is invalid");
|
|
406
|
+
if (!Number.isSafeInteger(archiveSize) || archiveSize < 1 || archiveSize > MAX_X_ZIP_ARCHIVE_BYTES) throw new Error("X archive size is invalid");
|
|
407
|
+
const directory = directoryFromEocd(descriptor, archiveSize);
|
|
408
|
+
const entries = centralEntries(descriptor, directory, archiveSize);
|
|
409
|
+
const ranges = localRanges(descriptor, entries, directory.offset);
|
|
410
|
+
const selected: ExtractedZipMember[] = [];
|
|
411
|
+
for (const entry of entries) {
|
|
412
|
+
if (!entry.selected) continue;
|
|
413
|
+
const range = ranges.get(entry.localHeaderOffset);
|
|
414
|
+
if (range === undefined) throw new Error("X ZIP selected member has no validated local range");
|
|
415
|
+
selected.push(readSelected(descriptor, entry, range));
|
|
416
|
+
}
|
|
417
|
+
if (selected.length === 0) throw new Error("archive contains no supported data/tweets*.js member");
|
|
418
|
+
return selected.sort((left, right) => left.memberName.localeCompare(right.memberName, "en"));
|
|
419
|
+
}
|
|
@@ -80,7 +80,7 @@ message entries in agent context, call X, or fetch media. The Message Like Me
|
|
|
80
80
|
CLI owns that strict offline parsing and preserves exact archive provenance.
|
|
81
81
|
This source covers archive direct messages, not X Chat. A separate, bounded
|
|
82
82
|
public-post evidence workflow is available through `$ensoul`'s shipped
|
|
83
|
-
`scripts/
|
|
83
|
+
`bun scripts/prepare-x-archive.ts`; that allowlisted adapter opens only authored
|
|
84
84
|
public-post members and must not be used to inspect direct-message data.
|
|
85
85
|
|
|
86
86
|
If the same X account is already represented by a Beeper source, inspect the
|
|
@@ -116,6 +116,13 @@ messagelikeme ingest bundle --input <absolute-private-whatsapp-bundle> --json
|
|
|
116
116
|
Do not request or handle Wacli session state or WhatsApp linked-device
|
|
117
117
|
authentication, call Wacli, inspect its database, synchronize the account, or
|
|
118
118
|
open bundle records in agent context. Wrench owns those provider operations.
|
|
119
|
+
The Wrench v0.16.5/Wacli v0.15.0 producer omits every reaction-shaped row and
|
|
120
|
+
reports `reaction-state-unproven` when it encounters one because current active
|
|
121
|
+
or removed state is not durable. Treat that warning, and an empty reaction
|
|
122
|
+
artifact from this producer, as unobservable reaction behavior. Do not infer
|
|
123
|
+
that the user avoided reactions or compare a native WhatsApp reaction ratio.
|
|
124
|
+
The v2 wire contract can parse a proven reaction record, but this exact producer
|
|
125
|
+
does not supply one.
|
|
119
126
|
If the CLI reports an exact existing Beeper WhatsApp account, inspect the
|
|
120
127
|
redacted source inventory and pass `--overlap-source <source-id>` only when the
|
|
121
128
|
user intends that reconciliation. The CLI must prove exact self and direct-peer
|
|
@@ -21,7 +21,9 @@ sensitive local data.
|
|
|
21
21
|
synchronize a linked device, inspect its database, or expose exact JIDs.
|
|
22
22
|
Wrench owns that provider boundary. `--overlap-source` requires explicit
|
|
23
23
|
intent and exact CLI proof; it never authorizes fuzzy account or contact
|
|
24
|
-
matching.
|
|
24
|
+
matching. The Wrench v0.16.5 producer omits reaction-shaped Wacli rows with
|
|
25
|
+
`reaction-state-unproven`; never turn that missing evidence into a claim that
|
|
26
|
+
no reactions occurred.
|
|
25
27
|
- Treat a caller-owned X data archive ZIP as private source evidence. Pass only
|
|
26
28
|
its explicit absolute path to `ingest x-archive`; do not extract it, evaluate
|
|
27
29
|
archive JavaScript, fetch linked media, or open its message entries in agent
|