@commercelayer/agent-cli 0.5.2 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +54 -6
- package/dist/api/client.js +39 -1
- package/dist/api/stub/http.js +2 -2
- package/dist/api/stub/management.js +19 -2
- package/dist/api/stub/messages.js +126 -0
- package/dist/api/stub/orders.js +286 -24
- package/dist/bin.js +30 -6
- package/dist/catalog.js +61 -19
- package/dist/commands.js +470 -7
- package/dist/cursor.js +31 -0
- package/dist/files.js +5 -3
- package/dist/inbox/cursor.js +11 -0
- package/dist/inbox/fake.js +95 -0
- package/dist/inbox/imap.js +139 -0
- package/dist/inbox/mailbox.js +25 -0
- package/dist/inbox/message.js +88 -0
- package/dist/inspect.js +230 -0
- package/dist/ops.js +190 -0
- package/dist/orderRequest.js +141 -0
- package/dist/output.js +7 -3
- package/dist/store.js +2 -0
- package/dist/web.js +316 -0
- package/dist/website.js +390 -0
- package/package.json +3 -1
package/dist/files.js
CHANGED
|
@@ -53,12 +53,14 @@ export function requireCredentials() {
|
|
|
53
53
|
return c;
|
|
54
54
|
}
|
|
55
55
|
/** Make sure .gitignore lists the credentials file. Returns true if it changed. */
|
|
56
|
-
export
|
|
56
|
+
export const ignoreCredentials = () => ignorePath(paths.credentials);
|
|
57
|
+
/** Add a path to .gitignore unless it is there; whether it was added. */
|
|
58
|
+
export function ignorePath(file) {
|
|
57
59
|
const existing = fs.existsSync('.gitignore') ? fs.readFileSync('.gitignore', 'utf8') : '';
|
|
58
|
-
if (existing.split('\n').includes(
|
|
60
|
+
if (existing.split('\n').includes(file))
|
|
59
61
|
return false;
|
|
60
62
|
const sep = existing && !existing.endsWith('\n') ? '\n' : '';
|
|
61
|
-
fs.appendFileSync('.gitignore', `${sep}${
|
|
63
|
+
fs.appendFileSync('.gitignore', `${sep}${file}\n`);
|
|
62
64
|
return true;
|
|
63
65
|
}
|
|
64
66
|
/** The note that sends a future agent in this repo to the skills. No skill names: it never goes stale. */
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
// What the inbox commands remember between runs, in .commercelayer/inbox.json (gitignored):
|
|
2
|
+
// when the inbox was last listed, and the messages already pulled, by Message-ID (or by id
|
|
3
|
+
// when a message has none), so inbox list marks only the others as new.
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import { jsonCursor, MAX_SEEN, strings } from '../cursor.js';
|
|
6
|
+
import { paths } from '../files.js';
|
|
7
|
+
const cursor = () => jsonCursor(path.join(paths.dir, 'inbox.json'), (c) => ({ last_check: typeof c?.last_check === 'string' ? c.last_check : null, seen: strings(c?.seen) }));
|
|
8
|
+
export const readInboxCursor = () => cursor().read();
|
|
9
|
+
export const writeInboxCursor = (c) => cursor().write({ last_check: c.last_check, seen: c.seen.slice(0, MAX_SEEN) });
|
|
10
|
+
/** The key a message is remembered by. */
|
|
11
|
+
export const seenKey = (m) => m.message_id ?? `id:${m.id}`;
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
// A mailbox for tests (C11R_IMAP_FAKE=<dir>): the .eml files in the folder are the messages,
|
|
2
|
+
// their id the file name's number (or the position), and flags.json keeps what IMAP would: seen,
|
|
3
|
+
// labels, the folder a message was moved to. "gmail": true in flags.json makes labels Gmail
|
|
4
|
+
// labels; "keywords": false makes a server with neither, which can only move.
|
|
5
|
+
import fs from 'node:fs';
|
|
6
|
+
import path from 'node:path';
|
|
7
|
+
import { fail } from '../output.js';
|
|
8
|
+
import { keywordOk } from './mailbox.js';
|
|
9
|
+
import { parseMessage } from './message.js';
|
|
10
|
+
export function openFake(dir, folder) {
|
|
11
|
+
const stateFile = path.join(dir, 'flags.json');
|
|
12
|
+
const read = () => {
|
|
13
|
+
const s = fs.existsSync(stateFile) ? JSON.parse(fs.readFileSync(stateFile, 'utf8')) : {};
|
|
14
|
+
return { ...s, messages: s.messages ?? {} };
|
|
15
|
+
};
|
|
16
|
+
const save = (s) => fs.writeFileSync(stateFile, JSON.stringify(s, null, 2) + '\n');
|
|
17
|
+
const files = () => fs
|
|
18
|
+
.readdirSync(dir)
|
|
19
|
+
.filter((f) => f.endsWith('.eml'))
|
|
20
|
+
.sort()
|
|
21
|
+
.map((f, i) => ({ id: /^\d+\.eml$/.test(f) ? String(Number(f.slice(0, -4))) : String(i + 1), file: path.join(dir, f) }));
|
|
22
|
+
const here = (s) => files().filter((m) => (s.messages[m.id]?.folder ?? 'INBOX') === folder);
|
|
23
|
+
const one = (id) => {
|
|
24
|
+
const s = read();
|
|
25
|
+
const m = here(s).find((x) => x.id === id);
|
|
26
|
+
return { s, m, st: (s.messages[id] ??= { flags: [], labels: [] }) };
|
|
27
|
+
};
|
|
28
|
+
const labelsOf = (s, id) => (s.gmail ? (s.messages[id]?.labels ?? []) : (s.messages[id]?.flags ?? []).filter((f) => !f.startsWith('\\')));
|
|
29
|
+
return {
|
|
30
|
+
host: 'fake',
|
|
31
|
+
user: 'orders@example.com',
|
|
32
|
+
folder,
|
|
33
|
+
async check() {
|
|
34
|
+
const s = read();
|
|
35
|
+
const all = here(s);
|
|
36
|
+
return { exists: all.length, unseen: all.filter((m) => !s.messages[m.id]?.flags.includes('\\Seen')).length };
|
|
37
|
+
},
|
|
38
|
+
async list({ unseen, since, limit }) {
|
|
39
|
+
const s = read();
|
|
40
|
+
const out = [];
|
|
41
|
+
for (const m of here(s)) {
|
|
42
|
+
const seen = s.messages[m.id]?.flags.includes('\\Seen') ?? false;
|
|
43
|
+
if (unseen && seen)
|
|
44
|
+
continue;
|
|
45
|
+
const p = await parseMessage(fs.readFileSync(m.file));
|
|
46
|
+
if (since && p.date && Date.parse(p.date) < since.getTime())
|
|
47
|
+
continue;
|
|
48
|
+
out.push({ id: m.id, message_id: p.message_id, from: p.from, subject: p.subject, date: p.date, seen, labels: labelsOf(s, m.id), size: fs.statSync(m.file).size });
|
|
49
|
+
}
|
|
50
|
+
return out.sort((a, b) => Number(b.id) - Number(a.id)).slice(0, limit);
|
|
51
|
+
},
|
|
52
|
+
async fetch(id) {
|
|
53
|
+
const { s, m } = one(id);
|
|
54
|
+
return m ? { raw: fs.readFileSync(m.file), labels: labelsOf(s, id) } : null;
|
|
55
|
+
},
|
|
56
|
+
async tag(id, t) {
|
|
57
|
+
const { s, m, st } = one(id);
|
|
58
|
+
if (!m)
|
|
59
|
+
fail('not_found', `There is no message ${id} in ${folder}.`, 'Run `c11r inbox list` for the ids.');
|
|
60
|
+
const r = { id, labels_as: null, labeled: [], unlabeled: [], seen: false };
|
|
61
|
+
if (t.label.length || t.unlabel.length) {
|
|
62
|
+
if (s.gmail) {
|
|
63
|
+
r.labels_as = 'gmail_label';
|
|
64
|
+
st.labels = [...new Set([...st.labels, ...t.label])].filter((l) => !t.unlabel.includes(l));
|
|
65
|
+
}
|
|
66
|
+
else if (s.keywords !== false) {
|
|
67
|
+
const bad = [...t.label, ...t.unlabel].find((l) => !keywordOk(l));
|
|
68
|
+
if (bad)
|
|
69
|
+
fail('invalid_argument', `${bad} cannot be a keyword on fake: no spaces, parentheses, quotes or backslashes.`, 'Use a one-word label (ordered, order-pending), or --move <folder>.');
|
|
70
|
+
r.labels_as = 'keyword';
|
|
71
|
+
st.flags = [...new Set([...st.flags, ...t.label])].filter((l) => !t.unlabel.includes(l));
|
|
72
|
+
}
|
|
73
|
+
else if (!t.move)
|
|
74
|
+
fail('labels_unsupported', `fake keeps no labels or keyword flags in ${folder}.`, 'Move the email to a folder instead: c11r inbox tag <id> --move <folder> [--seen].');
|
|
75
|
+
else
|
|
76
|
+
r.notes = [`fake keeps no labels or keywords: the email was moved to ${t.move} instead${t.unlabel.length ? ', and there was nothing to remove' : ''}.`];
|
|
77
|
+
if (r.labels_as) {
|
|
78
|
+
r.labeled = t.label;
|
|
79
|
+
r.unlabeled = t.unlabel;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
if (t.seen) {
|
|
83
|
+
st.flags = [...new Set([...st.flags, '\\Seen'])];
|
|
84
|
+
r.seen = true;
|
|
85
|
+
}
|
|
86
|
+
if (t.move) {
|
|
87
|
+
st.folder = t.move;
|
|
88
|
+
r.moved_to = t.move;
|
|
89
|
+
}
|
|
90
|
+
save(s);
|
|
91
|
+
return r;
|
|
92
|
+
},
|
|
93
|
+
async close() { },
|
|
94
|
+
};
|
|
95
|
+
}
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
// The inbox over IMAP with imapflow, loaded only here: the other commands never pay for it.
|
|
2
|
+
// Gmail keeps labels (X-GM-EXT-1); other servers keep keyword flags when the folder allows any
|
|
3
|
+
// (\*); a server with neither can only move the message to a folder.
|
|
4
|
+
import { fail } from '../output.js';
|
|
5
|
+
import { keywordOk } from './mailbox.js';
|
|
6
|
+
const who = (list) => {
|
|
7
|
+
const a = list?.[0];
|
|
8
|
+
if (!a?.address)
|
|
9
|
+
return null;
|
|
10
|
+
return a.name ? `${a.name} <${a.address}>` : a.address;
|
|
11
|
+
};
|
|
12
|
+
export async function openImap(s) {
|
|
13
|
+
const { ImapFlow } = await import('imapflow');
|
|
14
|
+
// What an error may say: never the password, even if a server echoes it.
|
|
15
|
+
const clean = (msg) => (s.password ? msg.split(s.password).join('***') : msg);
|
|
16
|
+
const unreachable = (e) => {
|
|
17
|
+
const err = e;
|
|
18
|
+
const reason = clean(err.responseText || err.message || String(e));
|
|
19
|
+
const auth = err.authenticationFailed || /auth|credentials|login/i.test(reason);
|
|
20
|
+
fail('inbox_unreachable', `Could not open ${s.folder} on ${s.host}:${s.port} as ${s.user}: ${reason}`, auth
|
|
21
|
+
? 'Check C11R_IMAP_USER and C11R_IMAP_PASSWORD in .commercelayer/credentials with the human: Gmail needs IMAP on and an app password (https://myaccount.google.com/apppasswords), not the account password.'
|
|
22
|
+
: 'Check C11R_IMAP_HOST and C11R_IMAP_PORT (993) in .commercelayer/credentials, and that this machine is online.');
|
|
23
|
+
};
|
|
24
|
+
const client = new ImapFlow({ host: s.host, port: s.port, secure: s.port !== 143, auth: { user: s.user, pass: s.password }, logger: false });
|
|
25
|
+
// An error after connecting (a dropped socket) must not crash the process.
|
|
26
|
+
client.on('error', () => { });
|
|
27
|
+
try {
|
|
28
|
+
await client.connect();
|
|
29
|
+
}
|
|
30
|
+
catch (e) {
|
|
31
|
+
unreachable(e);
|
|
32
|
+
}
|
|
33
|
+
let lock;
|
|
34
|
+
try {
|
|
35
|
+
lock = await client.getMailboxLock(s.folder);
|
|
36
|
+
}
|
|
37
|
+
catch (e) {
|
|
38
|
+
await client.logout().catch(() => { });
|
|
39
|
+
fail('inbox_unreachable', `There is no folder ${s.folder} on ${s.host} for ${s.user}: ${clean(e.message)}`, 'Set C11R_IMAP_FOLDER in .commercelayer/credentials to an existing folder, or leave it out for INBOX.');
|
|
40
|
+
}
|
|
41
|
+
const gmail = client.capabilities.has('X-GM-EXT-1');
|
|
42
|
+
const mailbox = () => (client.mailbox ? client.mailbox : undefined);
|
|
43
|
+
const envelopeOf = (m) => ({
|
|
44
|
+
id: String(m.uid),
|
|
45
|
+
message_id: m.envelope?.messageId ?? null,
|
|
46
|
+
from: who(m.envelope?.from),
|
|
47
|
+
subject: m.envelope?.subject ?? null,
|
|
48
|
+
date: m.envelope?.date ? new Date(m.envelope.date).toISOString() : null,
|
|
49
|
+
seen: m.flags?.has('\\Seen') ?? false,
|
|
50
|
+
labels: gmail ? [...(m.labels ?? [])] : [...(m.flags ?? [])].filter((f) => !f.startsWith('\\') && !f.startsWith('$')),
|
|
51
|
+
...(m.size ? { size: m.size } : {}),
|
|
52
|
+
});
|
|
53
|
+
const uidOf = (id) => {
|
|
54
|
+
if (!/^\d+$/.test(id))
|
|
55
|
+
fail('invalid_argument', `${id} is not a message id.`, 'Use the id inbox list prints (a number).');
|
|
56
|
+
return id;
|
|
57
|
+
};
|
|
58
|
+
return {
|
|
59
|
+
host: s.host,
|
|
60
|
+
user: s.user,
|
|
61
|
+
folder: s.folder,
|
|
62
|
+
async check() {
|
|
63
|
+
const st = await client.status(s.folder, { messages: true, unseen: true });
|
|
64
|
+
return { exists: st ? (st.messages ?? 0) : 0, unseen: st ? (st.unseen ?? 0) : 0 };
|
|
65
|
+
},
|
|
66
|
+
async list({ unseen, since, limit }) {
|
|
67
|
+
const query = unseen ? { seen: false } : { all: true };
|
|
68
|
+
if (since)
|
|
69
|
+
query.since = since;
|
|
70
|
+
const found = await client.search(query, { uid: true });
|
|
71
|
+
const uids = (Array.isArray(found) ? found : []).sort((a, b) => b - a).slice(0, limit);
|
|
72
|
+
if (!uids.length)
|
|
73
|
+
return [];
|
|
74
|
+
const out = [];
|
|
75
|
+
for await (const m of client.fetch(uids.join(','), { uid: true, envelope: true, flags: true, size: true, ...(gmail ? { labels: true } : {}) }, { uid: true }))
|
|
76
|
+
out.push(envelopeOf(m));
|
|
77
|
+
return out.sort((a, b) => Number(b.id) - Number(a.id));
|
|
78
|
+
},
|
|
79
|
+
async fetch(id) {
|
|
80
|
+
const m = await client.fetchOne(uidOf(id), { uid: true, source: true, flags: true, ...(gmail ? { labels: true } : {}) }, { uid: true });
|
|
81
|
+
if (!m || !m.source)
|
|
82
|
+
return null;
|
|
83
|
+
return { raw: m.source, labels: envelopeOf(m).labels };
|
|
84
|
+
},
|
|
85
|
+
async tag(id, t) {
|
|
86
|
+
const uid = uidOf(id);
|
|
87
|
+
if (!(await client.fetchOne(uid, { uid: true }, { uid: true })))
|
|
88
|
+
fail('not_found', `There is no message ${id} in ${s.folder}.`, 'Run `c11r inbox list` for the ids.');
|
|
89
|
+
const result = { id, labels_as: null, labeled: [], unlabeled: [], seen: false };
|
|
90
|
+
const notes = [];
|
|
91
|
+
if (t.label.length || t.unlabel.length) {
|
|
92
|
+
const keywords = mailbox()?.permanentFlags?.has('\\*') ?? false;
|
|
93
|
+
if (gmail) {
|
|
94
|
+
result.labels_as = 'gmail_label';
|
|
95
|
+
if (t.label.length)
|
|
96
|
+
await client.messageFlagsAdd(uid, t.label, { uid: true, useLabels: true });
|
|
97
|
+
if (t.unlabel.length)
|
|
98
|
+
await client.messageFlagsRemove(uid, t.unlabel, { uid: true, useLabels: true });
|
|
99
|
+
}
|
|
100
|
+
else if (keywords) {
|
|
101
|
+
const bad = [...t.label, ...t.unlabel].find((l) => !keywordOk(l));
|
|
102
|
+
if (bad)
|
|
103
|
+
fail('invalid_argument', `${bad} cannot be a keyword on ${s.host}: no spaces, parentheses, quotes or backslashes.`, 'Use a one-word label (ordered, order-pending), or --move <folder>.');
|
|
104
|
+
result.labels_as = 'keyword';
|
|
105
|
+
if (t.label.length)
|
|
106
|
+
await client.messageFlagsAdd(uid, t.label, { uid: true });
|
|
107
|
+
if (t.unlabel.length)
|
|
108
|
+
await client.messageFlagsRemove(uid, t.unlabel, { uid: true });
|
|
109
|
+
}
|
|
110
|
+
else if (!t.move) {
|
|
111
|
+
fail('labels_unsupported', `${s.host} keeps no labels or keyword flags in ${s.folder}.`, 'Move the email to a folder instead: c11r inbox tag <id> --move <folder> [--seen].');
|
|
112
|
+
}
|
|
113
|
+
else
|
|
114
|
+
notes.push(`${s.host} keeps no labels or keywords: the email was moved to ${t.move} instead${t.unlabel.length ? ', and there was nothing to remove' : ''}.`);
|
|
115
|
+
if (result.labels_as) {
|
|
116
|
+
result.labeled = t.label;
|
|
117
|
+
result.unlabeled = t.unlabel;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
if (t.seen) {
|
|
121
|
+
await client.messageFlagsAdd(uid, ['\\Seen'], { uid: true });
|
|
122
|
+
result.seen = true;
|
|
123
|
+
}
|
|
124
|
+
// Last: once moved, the message has another id in another folder.
|
|
125
|
+
if (t.move) {
|
|
126
|
+
if (!(await client.messageMove(uid, t.move, { uid: true })))
|
|
127
|
+
fail('inbox_unreachable', `${s.host} did not move message ${id} to ${t.move}.`, 'Check that the folder exists (create it in the mail app), then try again.');
|
|
128
|
+
result.moved_to = t.move;
|
|
129
|
+
}
|
|
130
|
+
if (notes.length)
|
|
131
|
+
result.notes = notes;
|
|
132
|
+
return result;
|
|
133
|
+
},
|
|
134
|
+
async close() {
|
|
135
|
+
lock.release();
|
|
136
|
+
await client.logout().catch(() => { });
|
|
137
|
+
},
|
|
138
|
+
};
|
|
139
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
// The inbox the merchant receives orders in, over IMAP: what inbox check, list, pull and tag
|
|
2
|
+
// work on. The connection details are in .commercelayer/credentials (gitignored) or the
|
|
3
|
+
// environment; the password is never printed, not even in an error. C11R_IMAP_FAKE=<dir> reads
|
|
4
|
+
// a folder of .eml files instead (tests).
|
|
5
|
+
import { credential } from '../files.js';
|
|
6
|
+
import { fail } from '../output.js';
|
|
7
|
+
const CREDENTIALS_HINT = 'Ask the human to add them to .commercelayer/credentials (gitignored), one per line: C11R_IMAP_HOST=imap.gmail.com, C11R_IMAP_USER=<their address>, C11R_IMAP_PASSWORD=<an app password>, and optionally C11R_IMAP_PORT (993) and C11R_IMAP_FOLDER (INBOX). For Gmail: turn on 2-Step Verification, then create an app password at https://myaccount.google.com/apppasswords; never the account password. They write the password into the file themselves: never ask for it in the chat.';
|
|
8
|
+
export function imapSettings() {
|
|
9
|
+
const missing = ['C11R_IMAP_HOST', 'C11R_IMAP_USER', 'C11R_IMAP_PASSWORD'].filter((k) => !credential(k));
|
|
10
|
+
if (missing.length)
|
|
11
|
+
fail('missing_credentials', `The inbox is not set up: ${missing.join(', ')} ${missing.length === 1 ? 'is' : 'are'} missing.`, CREDENTIALS_HINT);
|
|
12
|
+
const rawPort = credential('C11R_IMAP_PORT') ?? '993';
|
|
13
|
+
const port = Number(rawPort);
|
|
14
|
+
if (!Number.isInteger(port) || port < 1 || port > 65535)
|
|
15
|
+
fail('invalid_credentials', `C11R_IMAP_PORT is ${rawPort}, not a port.`, 'Use 993 (IMAP over TLS), the usual one, or leave it out.');
|
|
16
|
+
return { host: credential('C11R_IMAP_HOST'), port, user: credential('C11R_IMAP_USER'), password: credential('C11R_IMAP_PASSWORD'), folder: credential('C11R_IMAP_FOLDER') ?? 'INBOX' };
|
|
17
|
+
}
|
|
18
|
+
export async function openMailbox() {
|
|
19
|
+
const fake = process.env.C11R_IMAP_FAKE;
|
|
20
|
+
if (fake)
|
|
21
|
+
return (await import('./fake.js')).openFake(fake, credential('C11R_IMAP_FOLDER') ?? 'INBOX');
|
|
22
|
+
return (await import('./imap.js')).openImap(imapSettings());
|
|
23
|
+
}
|
|
24
|
+
/** An IMAP keyword is one atom: no spaces, parentheses, quotes or backslashes. */
|
|
25
|
+
export const keywordOk = (name) => /^[^\s(){}%*"\\\]]+$/.test(name);
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
// One email as the agent reads it: parsed with postal-mime (loaded only here, on first use) and
|
|
2
|
+
// written to .commercelayer/inbox/<id>/: message.json (headers, text, the attachments' files),
|
|
3
|
+
// body.txt (the text, or the HTML as text) and attachments/ with each file as it came. The
|
|
4
|
+
// CLI does not read PDFs or sheets: the agent opens them with its own abilities.
|
|
5
|
+
import fs from 'node:fs';
|
|
6
|
+
import path from 'node:path';
|
|
7
|
+
import { ignorePath, paths } from '../files.js';
|
|
8
|
+
export const MAX_ATTACHMENT = 25 * 1024 * 1024;
|
|
9
|
+
export const inboxDir = () => path.join(paths.dir, 'inbox');
|
|
10
|
+
const who = (a) => (a.group ? a.group.flatMap(who) : a.address ? [a.name ? `${a.name} <${a.address}>` : a.address] : []);
|
|
11
|
+
/** The HTML of a message as plain text, for one with no text part. */
|
|
12
|
+
export function htmlText(html) {
|
|
13
|
+
return html
|
|
14
|
+
.replace(/<(script|style|head)[^>]*>[\s\S]*?<\/\1>/gi, '')
|
|
15
|
+
.replace(/<br\s*\/?>/gi, '\n')
|
|
16
|
+
.replace(/<\/(p|div|tr|li|h[1-6]|table)>/gi, '\n')
|
|
17
|
+
.replace(/<(td|th)[^>]*>/gi, '\t')
|
|
18
|
+
.replace(/<[^>]+>/g, '')
|
|
19
|
+
.replace(/ /g, ' ')
|
|
20
|
+
.replace(/</g, '<')
|
|
21
|
+
.replace(/>/g, '>')
|
|
22
|
+
.replace(/"/g, '"')
|
|
23
|
+
.replace(/'/g, "'")
|
|
24
|
+
.replace(/&/g, '&')
|
|
25
|
+
.replace(/[ \t]+\n/g, '\n')
|
|
26
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
27
|
+
.trim();
|
|
28
|
+
}
|
|
29
|
+
export async function parseMessage(raw) {
|
|
30
|
+
const { default: PostalMime } = await import('postal-mime');
|
|
31
|
+
const m = await PostalMime.parse(raw, { attachmentEncoding: 'arraybuffer' });
|
|
32
|
+
const text = m.text?.trim() ? m.text : m.html ? htmlText(m.html) : '';
|
|
33
|
+
return {
|
|
34
|
+
message_id: m.messageId ?? null,
|
|
35
|
+
from: m.from ? (who(m.from)[0] ?? null) : null,
|
|
36
|
+
to: (m.to ?? []).flatMap((a) => who(a)),
|
|
37
|
+
subject: m.subject ?? null,
|
|
38
|
+
date: m.date ?? null,
|
|
39
|
+
text,
|
|
40
|
+
...(m.html ? { html: m.html } : {}),
|
|
41
|
+
attachments: m.attachments.map((a) => ({
|
|
42
|
+
name: a.filename,
|
|
43
|
+
content_type: a.mimeType,
|
|
44
|
+
bytes: typeof a.content === 'string' ? new TextEncoder().encode(a.content) : new Uint8Array(a.content),
|
|
45
|
+
inline: a.related === true || (a.disposition === 'inline' && !!a.contentId && a.mimeType.startsWith('image/')),
|
|
46
|
+
})),
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
const EXT = { 'application/pdf': 'pdf', 'text/plain': 'txt', 'text/csv': 'csv', 'image/png': 'png', 'image/jpeg': 'jpg', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': 'xlsx', 'application/vnd.ms-excel': 'xls', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': 'docx', 'application/msword': 'doc' };
|
|
50
|
+
/** A file name safe on every system: no folders, no control characters, never empty. */
|
|
51
|
+
export function safeName(name, i, type) {
|
|
52
|
+
// Slashes become underscores: "PO 2026/114.pdf" keeps its whole name and stays in its folder.
|
|
53
|
+
const base = (name ?? '').replace(/[/\\\u0000-\u001f<>:"|?*]+/g, '_').replace(/^[._]+/, '').trim().slice(0, 120);
|
|
54
|
+
return base || `attachment-${i + 1}${EXT[type] ? `.${EXT[type]}` : ''}`;
|
|
55
|
+
}
|
|
56
|
+
/** Write a parsed message under .commercelayer/inbox/<id>/; the folder is gitignored on first use. */
|
|
57
|
+
export function writeMessage(id, m, labels) {
|
|
58
|
+
const root = inboxDir();
|
|
59
|
+
if (!fs.existsSync(root))
|
|
60
|
+
ignorePath(`${root}/`);
|
|
61
|
+
const dir = path.join(root, id.replace(/[^A-Za-z0-9._-]+/g, '_'));
|
|
62
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
63
|
+
const warnings = [];
|
|
64
|
+
const attachments = [];
|
|
65
|
+
const used = new Set();
|
|
66
|
+
m.attachments.forEach((a, i) => {
|
|
67
|
+
if (a.inline)
|
|
68
|
+
return;
|
|
69
|
+
if (a.bytes.length > MAX_ATTACHMENT) {
|
|
70
|
+
warnings.push(`${a.name ?? `attachment ${i + 1}`} is over 25 MB and was not saved: open the email in the mail app.`);
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
let name = safeName(a.name, i, a.content_type);
|
|
74
|
+
for (let n = 2; used.has(name.toLowerCase()); n++)
|
|
75
|
+
name = name.replace(/(?: \(\d+\))?(\.[^.]*)?$/, ` (${n})$1`);
|
|
76
|
+
used.add(name.toLowerCase());
|
|
77
|
+
fs.mkdirSync(path.join(dir, 'attachments'), { recursive: true });
|
|
78
|
+
const file = path.join(dir, 'attachments', name);
|
|
79
|
+
fs.writeFileSync(file, a.bytes);
|
|
80
|
+
attachments.push({ file, name: a.name, content_type: a.content_type, size: a.bytes.length });
|
|
81
|
+
});
|
|
82
|
+
const body = path.join(dir, 'body.txt');
|
|
83
|
+
fs.writeFileSync(body, m.text.endsWith('\n') ? m.text : `${m.text}\n`);
|
|
84
|
+
const message = path.join(dir, 'message.json');
|
|
85
|
+
const json = { id, message_id: m.message_id, from: m.from, to: m.to, subject: m.subject, date: m.date, labels, text: m.text, ...(m.html ? { html: m.html } : {}), attachments: attachments.map(({ file, name, content_type, size }) => ({ file, name, content_type, size })) };
|
|
86
|
+
fs.writeFileSync(message, JSON.stringify(json, null, 2) + '\n');
|
|
87
|
+
return { id, dir, message, body, subject: m.subject, from: m.from, message_id: m.message_id, attachments, ...(warnings.length ? { warnings } : {}) };
|
|
88
|
+
}
|
package/dist/inspect.js
ADDED
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
// c11r storefront inspect <url>: what a brand's website says about itself, for a demo store that
|
|
2
|
+
// looks like it: title, languages, the logo, the colours, the currencies and countries it sells
|
|
3
|
+
// in, the platform, and whether robots.txt lets catalog pull read its products. Read-only: it
|
|
4
|
+
// writes no file and downloads nothing to keep; the logo is used by its URL (hot-linked).
|
|
5
|
+
import { fail } from './output.js';
|
|
6
|
+
import { absoluteUrl, BudgetExceeded, hreflangs, isType, jsonLd, metas, oneLine, platformOf, robotsAllows, siteOrigin, tags, unreachable, Web } from './web.js';
|
|
7
|
+
const MAX_STYLESHEETS = 3;
|
|
8
|
+
const MAX_LOGOS = 5;
|
|
9
|
+
const MAX_COLORS = 5;
|
|
10
|
+
const hex6 = (h) => {
|
|
11
|
+
const x = h.replace('#', '').toLowerCase();
|
|
12
|
+
return `#${x.length === 3 ? [...x].map((c) => c + c).join('') : x}`;
|
|
13
|
+
};
|
|
14
|
+
const rgbHex = (r, g, b) => `#${[r, g, b].map((n) => Math.max(0, Math.min(255, Math.round(n))).toString(16).padStart(2, '0')).join('')}`;
|
|
15
|
+
/** Greys, white and black: a page's text and background, rarely its brand colour. */
|
|
16
|
+
const neutral = (hex) => {
|
|
17
|
+
const [r, g, b] = [1, 3, 5].map((i) => parseInt(hex.slice(i, i + 2), 16));
|
|
18
|
+
return Math.max(r, g, b) - Math.min(r, g, b) < 16;
|
|
19
|
+
};
|
|
20
|
+
/** Every colour written in CSS: #rgb, #rrggbb and rgb()/rgba(), counted. */
|
|
21
|
+
export function cssColors(css, into = new Map()) {
|
|
22
|
+
const add = (h) => into.set(h, (into.get(h) ?? 0) + 1);
|
|
23
|
+
for (const m of css.matchAll(/#([0-9a-f]{6}|[0-9a-f]{3})(?![0-9a-z_-])/gi))
|
|
24
|
+
add(hex6(m[1]));
|
|
25
|
+
for (const m of css.matchAll(/rgba?\(\s*(\d{1,3})[\s,]+(\d{1,3})[\s,]+(\d{1,3})(?:\s*[,/]\s*([\d.]+%?))?\s*\)/gi)) {
|
|
26
|
+
// A transparent colour is an overlay, not a colour of the brand.
|
|
27
|
+
if (m[4] !== undefined && parseFloat(m[4]) === 0)
|
|
28
|
+
continue;
|
|
29
|
+
add(rgbHex(Number(m[1]), Number(m[2]), Number(m[3])));
|
|
30
|
+
}
|
|
31
|
+
return into;
|
|
32
|
+
}
|
|
33
|
+
const num = (v) => {
|
|
34
|
+
const n = Number(String(v ?? '').replace(/px$/, ''));
|
|
35
|
+
return Number.isFinite(n) && n > 0 ? n : undefined;
|
|
36
|
+
};
|
|
37
|
+
/** Where the header is: the <header> element, else the top of the body. */
|
|
38
|
+
const headerOf = (html) => {
|
|
39
|
+
const h = html.match(/<header\b[\s\S]*?<\/header>/i)?.[0];
|
|
40
|
+
if (h)
|
|
41
|
+
return h;
|
|
42
|
+
const body = html.search(/<body\b/i);
|
|
43
|
+
return html.slice(Math.max(0, body), Math.max(0, body) + 20_000);
|
|
44
|
+
};
|
|
45
|
+
export function logoCandidates(html, base, siteTitle) {
|
|
46
|
+
const out = [];
|
|
47
|
+
for (const node of jsonLd(html)) {
|
|
48
|
+
if (!['Organization', 'Brand', 'OnlineStore', 'Store', 'Corporation', 'LocalBusiness', 'WebSite'].some((t) => isType(node, t)))
|
|
49
|
+
continue;
|
|
50
|
+
const l = Array.isArray(node.logo) ? node.logo[0] : node.logo;
|
|
51
|
+
const url = absoluteUrl(typeof l === 'string' ? l : (l?.url ?? l?.contentUrl), base);
|
|
52
|
+
if (url)
|
|
53
|
+
out.push({ url, source: 'json_ld', score: 4, ...(num(l?.width) ? { width: num(l.width) } : {}), ...(num(l?.height) ? { height: num(l.height) } : {}) });
|
|
54
|
+
}
|
|
55
|
+
const words = (siteTitle ?? '').toLowerCase();
|
|
56
|
+
const scan = (region, source, bonus) => {
|
|
57
|
+
for (const t of tags(region, 'img')) {
|
|
58
|
+
const a = t.attrs;
|
|
59
|
+
const src = a.src || a['data-src'] || (a.srcset ?? a['data-srcset'] ?? '').split(',')[0]?.trim().split(/\s+/)[0];
|
|
60
|
+
// Shopify themes leave {width} in lazy sources.
|
|
61
|
+
const url = absoluteUrl(src?.replace(/\{width\}/g, '600'), base);
|
|
62
|
+
if (!url)
|
|
63
|
+
continue;
|
|
64
|
+
const own = [a.class, a.id, a.alt, src].join(' ').toLowerCase();
|
|
65
|
+
// The element around it (a link or a wrapper named logo), not just anything near.
|
|
66
|
+
const parents = [...region.slice(Math.max(0, t.index - 600), t.index).matchAll(/<(?!img\b|\/)[a-z][^>]*>/gi)].slice(-2).map((m) => m[0]).join(' ').toLowerCase();
|
|
67
|
+
const alt = (a.alt ?? '').trim().toLowerCase();
|
|
68
|
+
const score = (own.includes('logo') ? 3 : 0) + (alt && words && (words.includes(alt) || alt.includes(words.split(/\s[|–—-]\s/)[0])) ? 2 : 0) + (parents.includes('logo') ? 1 : 0);
|
|
69
|
+
if (score === 0)
|
|
70
|
+
continue;
|
|
71
|
+
out.push({ url, source, score: score + bonus, ...(a.alt ? { alt: a.alt.trim() } : {}), ...(num(a.width) ? { width: num(a.width) } : {}), ...(num(a.height) ? { height: num(a.height) } : {}) });
|
|
72
|
+
}
|
|
73
|
+
};
|
|
74
|
+
const header = headerOf(html);
|
|
75
|
+
scan(header, 'header_img', 1);
|
|
76
|
+
scan(html, 'img', 0);
|
|
77
|
+
// An inline <svg> logo has no URL to link: say so, so the agent asks for a file or picks another.
|
|
78
|
+
for (const t of tags(header, 'svg')) {
|
|
79
|
+
const own = [t.attrs.class, t.attrs.id, t.attrs['aria-label']].join(' ').toLowerCase();
|
|
80
|
+
const titled = header.slice(t.index, t.index + 400).match(/<title>([^<]*)<\/title>/i)?.[1]?.toLowerCase();
|
|
81
|
+
if (!own.includes('logo') && !(titled && words && words.includes(titled.trim())))
|
|
82
|
+
continue;
|
|
83
|
+
out.push({ url: null, source: 'header_svg', score: 2, note: 'An inline SVG in the header: it has no URL to link. Use another candidate, or ask the human for the logo file (img/logo.svg).' });
|
|
84
|
+
break;
|
|
85
|
+
}
|
|
86
|
+
const seen = new Set();
|
|
87
|
+
return out
|
|
88
|
+
.sort((a, b) => b.score - a.score)
|
|
89
|
+
.filter((l) => {
|
|
90
|
+
const key = l.url ?? `svg:${l.source}`;
|
|
91
|
+
if (seen.has(key))
|
|
92
|
+
return false;
|
|
93
|
+
seen.add(key);
|
|
94
|
+
return true;
|
|
95
|
+
})
|
|
96
|
+
.slice(0, MAX_LOGOS)
|
|
97
|
+
.map(({ score: _, ...l }) => l);
|
|
98
|
+
}
|
|
99
|
+
const ISO_CURRENCY = /^[A-Z]{3}$/;
|
|
100
|
+
export function currenciesSeen(html, meta) {
|
|
101
|
+
const found = new Set();
|
|
102
|
+
const add = (c) => typeof c === 'string' && ISO_CURRENCY.test(c.trim().toUpperCase()) && found.add(c.trim().toUpperCase());
|
|
103
|
+
add(meta['og:price:currency']);
|
|
104
|
+
add(meta['product:price:currency']);
|
|
105
|
+
for (const m of html.matchAll(/Shopify\.currency\s*=\s*\{[^}]*"active"\s*:\s*"([A-Za-z]{3})"/g))
|
|
106
|
+
add(m[1]);
|
|
107
|
+
for (const m of html.matchAll(/"priceCurrency"\s*:\s*"([A-Za-z]{3})"/g))
|
|
108
|
+
add(m[1]);
|
|
109
|
+
for (const m of html.matchAll(/"currency_code"\s*:\s*"([A-Za-z]{3})"/g))
|
|
110
|
+
add(m[1]);
|
|
111
|
+
for (const m of html.matchAll(/data-currency(?:-code)?\s*=\s*["']([A-Za-z]{3})["']/g))
|
|
112
|
+
add(m[1]);
|
|
113
|
+
// A currency selector: <select name="currency"> with ISO codes as values.
|
|
114
|
+
for (const s of html.matchAll(/<select\b[^>]*name\s*=\s*["']?currency[^>]*>([\s\S]*?)<\/select>/gi))
|
|
115
|
+
for (const o of tags(s[1], 'option'))
|
|
116
|
+
add(o.attrs.value);
|
|
117
|
+
return [...found].sort();
|
|
118
|
+
}
|
|
119
|
+
export function countriesSeen(html, base) {
|
|
120
|
+
const found = new Set();
|
|
121
|
+
const add = (c) => typeof c === 'string' && /^[A-Z]{2}$/.test(c.trim()) && found.add(c.trim());
|
|
122
|
+
for (const h of hreflangs(html, base))
|
|
123
|
+
if (h.region && /^[A-Z]{2}$/.test(h.region))
|
|
124
|
+
found.add(h.region);
|
|
125
|
+
for (const m of html.matchAll(/Shopify\.country\s*=\s*"([A-Z]{2})"/g))
|
|
126
|
+
add(m[1]);
|
|
127
|
+
// A country selector: Shopify's localization form, or a <select name="country...">.
|
|
128
|
+
for (const f of html.matchAll(/<form\b[^>]*localization[^>]*>([\s\S]*?)<\/form>/gi))
|
|
129
|
+
for (const m of f[1].matchAll(/data-value\s*=\s*["']([A-Z]{2})["']/g))
|
|
130
|
+
add(m[1]);
|
|
131
|
+
for (const s of html.matchAll(/<select\b[^>]*name\s*=\s*["']?country[^>]*>([\s\S]*?)<\/select>/gi))
|
|
132
|
+
for (const o of tags(s[1], 'option'))
|
|
133
|
+
add(o.attrs.value);
|
|
134
|
+
return [...found].sort();
|
|
135
|
+
}
|
|
136
|
+
/** The paths catalog pull reads, by strategy: whether robots.txt lets c11r read them. */
|
|
137
|
+
const PRODUCT_PATHS = { shopify: ['/products.json', '/collections/all/products.json'], woocommerce: ['/wp-json/wc/store/v1/products'], structured_data: ['/sitemap.xml'] };
|
|
138
|
+
export async function inspect(raw) {
|
|
139
|
+
if (!raw)
|
|
140
|
+
fail('missing_argument', 'storefront inspect needs the website address.', 'Run c11r storefront inspect https://brand.example.');
|
|
141
|
+
const origin = siteOrigin(raw, 'The address');
|
|
142
|
+
const withScheme = /^[a-z][a-z0-9+.-]*:\/\//i.test(raw) ? raw : `https://${raw}`;
|
|
143
|
+
const start = absoluteUrl(withScheme, origin) ?? `${origin}/`;
|
|
144
|
+
const web = new Web();
|
|
145
|
+
let page;
|
|
146
|
+
try {
|
|
147
|
+
page = await web.get(start);
|
|
148
|
+
}
|
|
149
|
+
catch (e) {
|
|
150
|
+
if (e instanceof BudgetExceeded)
|
|
151
|
+
fail('site_unreachable', `${start} did not answer in time.`, 'Try again later.');
|
|
152
|
+
unreachable(start, e);
|
|
153
|
+
}
|
|
154
|
+
if (page === null)
|
|
155
|
+
fail('robots_disallowed', `${start}'s robots.txt does not let c11r read that page.`, 'c11r does not go around robots.txt. Ask the human for the logo, colours and products instead (a feed or a sheet for catalog pull).');
|
|
156
|
+
if (page.status === 0)
|
|
157
|
+
fail('site_unreachable', `${start} did not answer within ${Math.round(web.timeout / 1000)} s.`, 'Check the address with the human (open it in a browser), and try again later.');
|
|
158
|
+
if (!page.ok)
|
|
159
|
+
fail('site_error', `${start} answered ${page.status}.`, 'Check the address with the human: open it in a browser.');
|
|
160
|
+
const html = page.text;
|
|
161
|
+
const base = page.url;
|
|
162
|
+
const finalOrigin = new URL(base).origin;
|
|
163
|
+
const meta = metas(html);
|
|
164
|
+
const title = oneLine(html.match(/<title\b[^>]*>([\s\S]*?)<\/title>/i)?.[1] ?? '') || null;
|
|
165
|
+
const siteName = meta['og:site_name'] || null;
|
|
166
|
+
const lang = html.match(/<html\b[^>]*\blang\s*=\s*["']?([A-Za-z]{2,3}(?:[-_][A-Za-z0-9]+)*)/)?.[1] ?? null;
|
|
167
|
+
const alternates = hreflangs(html, base);
|
|
168
|
+
const languages = [...new Set([lang?.slice(0, 2).toLowerCase(), ...alternates.map((h) => h.language)].filter((l) => !!l))];
|
|
169
|
+
const icons = tags(html, 'link')
|
|
170
|
+
.filter((t) => /\b(icon|apple-touch-icon|mask-icon)\b/i.test(t.attrs.rel ?? ''))
|
|
171
|
+
.map((t) => ({ url: absoluteUrl(t.attrs.href, base), rel: t.attrs.rel, ...(t.attrs.sizes ? { sizes: t.attrs.sizes } : {}), ...(t.attrs.type ? { type: t.attrs.type } : {}) }))
|
|
172
|
+
.filter((i) => i.url);
|
|
173
|
+
// Colours: theme-color first, then the CSS: inline <style>, style="" and the first linked stylesheets.
|
|
174
|
+
const counts = new Map();
|
|
175
|
+
for (const m of html.matchAll(/<style\b[^>]*>([\s\S]*?)<\/style>/gi))
|
|
176
|
+
cssColors(m[1], counts);
|
|
177
|
+
for (const m of html.matchAll(/\sstyle\s*=\s*"([^"]*)"/gi))
|
|
178
|
+
cssColors(m[1], counts);
|
|
179
|
+
const sheets = tags(html, 'link')
|
|
180
|
+
.filter((t) => /\bstylesheet\b/i.test(t.attrs.rel ?? '') && t.attrs.href)
|
|
181
|
+
.map((t) => absoluteUrl(t.attrs.href, base))
|
|
182
|
+
.filter((u) => !!u)
|
|
183
|
+
.slice(0, MAX_STYLESHEETS);
|
|
184
|
+
let stylesheetsRead = 0;
|
|
185
|
+
for (const url of sheets) {
|
|
186
|
+
try {
|
|
187
|
+
const css = await web.get(url, 'text/css,*/*;q=0.1');
|
|
188
|
+
if (css?.ok) {
|
|
189
|
+
cssColors(css.text, counts);
|
|
190
|
+
stylesheetsRead++;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
catch (e) {
|
|
194
|
+
if (e instanceof BudgetExceeded)
|
|
195
|
+
break;
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
const themeColor = /^#([0-9a-f]{3}|[0-9a-f]{6})$/i.test(meta['theme-color'] ?? '') ? hex6(meta['theme-color']) : null;
|
|
199
|
+
const colors = [
|
|
200
|
+
...(themeColor ? [{ hex: themeColor, source: 'theme_color', count: counts.get(themeColor) ?? 0, neutral: neutral(themeColor) }] : []),
|
|
201
|
+
...[...counts.entries()]
|
|
202
|
+
.filter(([h]) => h !== themeColor)
|
|
203
|
+
.sort((a, b) => b[1] - a[1])
|
|
204
|
+
.map(([hex, count]) => ({ hex, source: 'css', count, neutral: neutral(hex) })),
|
|
205
|
+
].slice(0, MAX_COLORS);
|
|
206
|
+
const robots = await web.robotsFor(finalOrigin).catch(() => ({ rules: [], sitemaps: [] }));
|
|
207
|
+
const paths = Object.fromEntries(Object.values(PRODUCT_PATHS).flat().map((p) => [p, robotsAllows(robots, p)]));
|
|
208
|
+
const platform = platformOf(html, page.headers);
|
|
209
|
+
const relevant = platform === 'unknown' ? Object.values(PRODUCT_PATHS).flat() : [...PRODUCT_PATHS[platform], ...PRODUCT_PATHS.structured_data];
|
|
210
|
+
return {
|
|
211
|
+
url: base,
|
|
212
|
+
title,
|
|
213
|
+
site_name: siteName,
|
|
214
|
+
description: meta.description || meta['og:description'] || null,
|
|
215
|
+
language: lang,
|
|
216
|
+
languages,
|
|
217
|
+
theme_color: themeColor,
|
|
218
|
+
icons,
|
|
219
|
+
og_image: absoluteUrl(meta['og:image:secure_url'] || meta['og:image'], base),
|
|
220
|
+
logo_candidates: logoCandidates(html, base, siteName ?? title),
|
|
221
|
+
colors,
|
|
222
|
+
currencies_seen: currenciesSeen(html, meta),
|
|
223
|
+
countries_seen: countriesSeen(html, base),
|
|
224
|
+
platform,
|
|
225
|
+
robots: { products_allowed: relevant.some((p) => paths[p]), paths },
|
|
226
|
+
stylesheets_read: stylesheetsRead,
|
|
227
|
+
requests: web.requests,
|
|
228
|
+
next: 'Pick theme.color from colors (the first one that is not neutral, usually) and theme.logo from logo_candidates (its url, hot-linked: nothing is copied), and show the human what you picked. Then catalog: { source: website, url: <this site> } in store.yml and c11r catalog pull.' + (relevant.some((p) => paths[p]) ? '' : ' robots.txt disallows the paths catalog pull reads: ask the human for a feed or a sheet instead.'),
|
|
229
|
+
};
|
|
230
|
+
}
|