@forwardimpact/outpost 3.12.0 → 3.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/bin/fit-outpost.js +3 -2
- package/package.json +6 -5
- package/src/agent-path.js +10 -6
- package/src/agent-runner.js +28 -24
- package/src/index.js +2 -3
- package/src/kb-manager.js +75 -20
- package/src/kb-validator.js +762 -0
- package/src/outpost.js +108 -39
- package/src/posture.js +3 -3
- package/src/privilege.js +11 -11
- package/src/scheduler.js +14 -12
- package/src/socket-server.js +19 -18
- package/src/spawn-env.js +7 -6
- package/src/state-manager.js +2 -2
- package/templates/.claude/agents/chief-of-staff.md +32 -24
- package/templates/.claude/agents/concierge.md +26 -17
- package/templates/.claude/agents/head-hunter.md +34 -26
- package/templates/.claude/agents/librarian.md +26 -18
- package/templates/.claude/agents/postman.md +26 -18
- package/templates/.claude/agents/recruiter.md +38 -27
- package/templates/.claude/skills/anarlog-follow/SKILL.md +91 -66
- package/templates/.claude/skills/anarlog-follow/references/coaching.md +22 -21
- package/templates/.claude/skills/anarlog-follow/references/sessions.md +52 -0
- package/templates/.claude/skills/anarlog-process/SKILL.md +108 -62
- package/templates/.claude/skills/anarlog-process/references/extraction.md +41 -29
- package/templates/.claude/skills/anarlog-process/references/sessions.md +92 -58
- package/templates/.claude/skills/anarlog-process/scripts/scan.mjs +402 -160
- package/templates/.claude/skills/candidate-report/SKILL.md +41 -36
- package/templates/.claude/skills/candidate-report/references/report-template.html +10 -10
- package/templates/.claude/skills/candidate-report/references/report.css +4 -4
- package/templates/.claude/skills/candidate-report/references/rubric.md +15 -14
- package/templates/.claude/skills/candidate-report/scripts/render-pdf.mjs +2 -2
- package/templates/.claude/skills/changelog/SKILL.md +122 -67
- package/templates/.claude/skills/deck-create/SKILL.md +90 -32
- package/templates/.claude/skills/deck-create/references/slide.css +4 -4
- package/templates/.claude/skills/deck-create/scripts/convert-to-pdf.mjs +6 -6
- package/templates/.claude/skills/deck-review/SKILL.md +63 -58
- package/templates/.claude/skills/deck-review/assets/slide-annotator.js +58 -49
- package/templates/.claude/skills/deck-summarize/SKILL.md +55 -45
- package/templates/.claude/skills/deck-summarize/references/brief-template.md +5 -4
- package/templates/.claude/skills/deck-summarize/scripts/extract-pptx.mjs +8 -7
- package/templates/.claude/skills/doc-collab/SKILL.md +39 -34
- package/templates/.claude/skills/doc-create/SKILL.md +27 -23
- package/templates/.claude/skills/doc-create/scripts/convert-to-pdf.mjs +6 -6
- package/templates/.claude/skills/draft-emails/SKILL.md +63 -53
- package/templates/.claude/skills/draft-emails/references/template.md +2 -2
- package/templates/.claude/skills/draft-emails/scripts/scan-emails.mjs +15 -8
- package/templates/.claude/skills/draft-emails/scripts/send-email.mjs +19 -11
- package/templates/.claude/skills/extract-entities/SKILL.md +76 -75
- package/templates/.claude/skills/extract-entities/references/TEMPLATES.md +6 -5
- package/templates/.claude/skills/extract-entities/references/conditions.md +20 -18
- package/templates/.claude/skills/extract-entities/references/content.md +10 -9
- package/templates/.claude/skills/extract-entities/references/links.md +37 -13
- package/templates/.claude/skills/extract-entities/references/recruitment.md +30 -24
- package/templates/.claude/skills/extract-entities/references/resolution.md +14 -14
- package/templates/.claude/skills/extract-entities/references/sources.md +9 -9
- package/templates/.claude/skills/extract-entities/references/templates-conditions.md +12 -6
- package/templates/.claude/skills/extract-entities/references/templates-people-orgs.md +29 -10
- package/templates/.claude/skills/extract-entities/references/templates-priorities.md +10 -4
- package/templates/.claude/skills/extract-entities/references/templates-projects-topics.md +19 -7
- package/templates/.claude/skills/extract-entities/scripts/state.mjs +9 -8
- package/templates/.claude/skills/meeting-prep/SKILL.md +42 -36
- package/templates/.claude/skills/organize-files/SKILL.md +27 -22
- package/templates/.claude/skills/organize-files/scripts/organize-by-type.mjs +5 -4
- package/templates/.claude/skills/organize-files/scripts/summarize.mjs +4 -4
- package/templates/.claude/skills/person-identify/SKILL.md +77 -27
- package/templates/.claude/skills/person-identify/scripts/identify.sh +125 -26
- package/templates/.claude/skills/person-lookup/SKILL.md +34 -30
- package/templates/.claude/skills/person-lookup/scripts/lookup.sh +46 -17
- package/templates/.claude/skills/req-assess/SKILL.md +44 -36
- package/templates/.claude/skills/req-assess/references/interview-template.md +7 -2
- package/templates/.claude/skills/req-assess/references/panel-template.md +7 -2
- package/templates/.claude/skills/req-assess/references/rubric.md +13 -13
- package/templates/.claude/skills/req-bundle/SKILL.md +164 -0
- package/templates/.claude/skills/req-bundle/references/matching.md +70 -0
- package/templates/.claude/skills/req-bundle/references/pdf-structure.md +74 -0
- package/templates/.claude/skills/req-bundle/scripts/split-bundle.mjs +377 -0
- package/templates/.claude/skills/req-decide/SKILL.md +52 -41
- package/templates/.claude/skills/req-decide/references/rubric.md +11 -10
- package/templates/.claude/skills/req-decide/references/template.md +2 -2
- package/templates/.claude/skills/req-forget/SKILL.md +58 -39
- package/templates/.claude/skills/req-forget/references/classify.md +15 -13
- package/templates/.claude/skills/req-forget/references/locations.md +27 -22
- package/templates/.claude/skills/req-forget/references/report-template.md +24 -16
- package/templates/.claude/skills/req-scan/SKILL.md +27 -25
- package/templates/.claude/skills/req-scan/references/fallbacks.md +9 -9
- package/templates/.claude/skills/req-scan/references/filters.md +13 -13
- package/templates/.claude/skills/req-scan/references/sources.md +7 -7
- package/templates/.claude/skills/req-scan/references/state.md +1 -1
- package/templates/.claude/skills/req-scan/references/template.md +11 -1
- package/templates/.claude/skills/req-scan/scripts/state.mjs +3 -2
- package/templates/.claude/skills/req-screen/SKILL.md +96 -50
- package/templates/.claude/skills/req-screen/references/rubric.md +46 -50
- package/templates/.claude/skills/req-screen/references/scoring.md +79 -0
- package/templates/.claude/skills/req-screen/references/template.md +29 -17
- package/templates/.claude/skills/req-track/SKILL.md +76 -76
- package/templates/.claude/skills/req-track/references/fields.md +31 -27
- package/templates/.claude/skills/req-track/references/overlays.md +42 -0
- package/templates/.claude/skills/req-track/references/signals.md +12 -12
- package/templates/.claude/skills/req-track/references/statuses.md +3 -2
- package/templates/.claude/skills/req-track/references/templates.md +28 -27
- package/templates/.claude/skills/req-workday/SKILL.md +69 -40
- package/templates/.claude/skills/req-workday/references/brief.md +81 -0
- package/templates/.claude/skills/req-workday/references/status-mapping.md +5 -5
- package/templates/.claude/skills/req-workday/references/templates.md +11 -78
- package/templates/.claude/skills/req-workday/references/xlsx-format.md +26 -7
- package/templates/.claude/skills/req-workday/scripts/parse-workday.mjs +113 -35
- package/templates/.claude/skills/send-chat/SKILL.md +58 -54
- package/templates/.claude/skills/sync-apple-calendar/SKILL.md +23 -19
- package/templates/.claude/skills/sync-apple-calendar/references/SCHEMA.md +7 -7
- package/templates/.claude/skills/sync-apple-calendar/scripts/query.mjs +6 -6
- package/templates/.claude/skills/sync-apple-calendar/scripts/sync.mjs +9 -8
- package/templates/.claude/skills/sync-apple-mail/SKILL.md +20 -14
- package/templates/.claude/skills/sync-apple-mail/references/SCHEMA.md +10 -10
- package/templates/.claude/skills/sync-apple-mail/scripts/parse-emlx.mjs +9 -9
- package/templates/.claude/skills/sync-apple-mail/scripts/sync-helpers.mjs +16 -13
- package/templates/.claude/skills/sync-apple-mail/scripts/sync.mjs +3 -3
- package/templates/.claude/skills/sync-teams/SKILL.md +46 -43
- package/templates/.claude/skills/sync-teams/scripts/idb-reader.mjs +22 -21
- package/templates/.claude/skills/sync-teams/scripts/leveldb-reader.mjs +4 -4
- package/templates/.claude/skills/sync-teams/scripts/sync.mjs +4 -4
- package/templates/.claude/skills/upstream-instructions/SKILL.md +43 -41
- package/templates/.claude/skills/upstream-instructions/references/examples.md +5 -5
- package/templates/CLAUDE.md +97 -85
- package/templates/MIGRATION.md +359 -0
- package/templates/registry.yaml +25 -0
- package/templates/.claude/skills/anarlog-follow/scripts/follow.mjs +0 -243
- package/templates/.claude/skills/anarlog-trim/SKILL.md +0 -183
|
@@ -6,9 +6,12 @@ compatibility: Requires macOS with Apple Mail configured and Full Disk Access gr
|
|
|
6
6
|
|
|
7
7
|
# Sync Apple Mail
|
|
8
8
|
|
|
9
|
+
Write tier: none (cache only)
|
|
10
|
+
Frontmatter: none
|
|
11
|
+
|
|
9
12
|
Sync email threads from the macOS Mail app's local SQLite database into
|
|
10
|
-
`~/.cache/fit/outpost/apple_mail/` as markdown files. This is an automated
|
|
11
|
-
|
|
13
|
+
`~/.cache/fit/outpost/apple_mail/` as markdown files. This is an automated skill
|
|
14
|
+
in the data pipeline. It ingests raw email data that other skills (like
|
|
12
15
|
`extract-entities`) consume downstream.
|
|
13
16
|
|
|
14
17
|
## Trigger
|
|
@@ -46,7 +49,7 @@ their email.
|
|
|
46
49
|
## Implementation
|
|
47
50
|
|
|
48
51
|
Run the sync as a single Node.js script with embedded SQLite. This avoids N+1
|
|
49
|
-
process invocations
|
|
52
|
+
process invocations. It also transforms all the data in one pass:
|
|
50
53
|
|
|
51
54
|
```text
|
|
52
55
|
node scripts/sync.mjs [--days N]
|
|
@@ -60,27 +63,29 @@ The script:
|
|
|
60
63
|
2. Loads last sync timestamp (or defaults to `--days` days ago for first sync)
|
|
61
64
|
3. Discovers the thread grouping column (`conversation_id` or `thread_id`)
|
|
62
65
|
4. Loads last-seen ROWID (or defaults to 0 for first sync)
|
|
63
|
-
5. Finds threads with new messages since last sync (up to 500)
|
|
64
|
-
timestamp and ROWID to catch late
|
|
65
|
-
delay may have `date_received` before the last sync timestamp
|
|
66
|
-
ROWID
|
|
66
|
+
5. Finds threads with new messages since last sync (up to 500). It uses both the
|
|
67
|
+
timestamp and the ROWID to catch emails that arrive late. An email downloaded
|
|
68
|
+
after a delay may have a `date_received` before the last sync timestamp. But
|
|
69
|
+
its ROWID is higher than the last-seen ROWID
|
|
67
70
|
6. For each thread: fetches messages, batch-fetches recipients and attachment
|
|
68
|
-
metadata, parses `.emlx` files for full email bodies (
|
|
71
|
+
metadata, parses `.emlx` files for full email bodies (with a fallback to
|
|
69
72
|
database summaries), copies attachment files to the output directory
|
|
70
73
|
7. Writes one markdown file per thread to `~/.cache/fit/outpost/apple_mail/`
|
|
71
74
|
8. Updates sync state (timestamp and max ROWID)
|
|
72
75
|
9. Reports summary (threads processed, files written)
|
|
73
76
|
|
|
74
77
|
The script imports `scripts/parse-emlx.mjs` to extract plain text bodies from
|
|
75
|
-
`.emlx` / `.partial.emlx` files
|
|
78
|
+
`.emlx` / `.partial.emlx` files. The parser strips the tags from HTML-only
|
|
79
|
+
emails.
|
|
76
80
|
|
|
77
81
|
## Database Schema
|
|
78
82
|
|
|
79
83
|
See [references/SCHEMA.md](references/SCHEMA.md) for the complete Apple Mail
|
|
80
|
-
SQLite schema
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
`message_id
|
|
84
|
+
SQLite schema. It gives the table structures, the column names, and the
|
|
85
|
+
important caveats. For example, `date_received` holds Unix timestamps. It does
|
|
86
|
+
not hold Core Data timestamps. `addresses.comment` holds display names. The
|
|
87
|
+
`recipients` columns are `message` and `address`. They are not `message_id` and
|
|
88
|
+
`address_id`.
|
|
84
89
|
|
|
85
90
|
## Output Format
|
|
86
91
|
|
|
@@ -139,7 +144,8 @@ Rules:
|
|
|
139
144
|
- `.emlx` parse error → fall back to database summary field
|
|
140
145
|
- HTML-only email → strip tags and use as plain text body (handled by
|
|
141
146
|
parse-emlx.mjs)
|
|
142
|
-
- `find` timeout → skip that message's body
|
|
147
|
+
- `find` timeout → skip that message's body and use the summary. The attachment
|
|
148
|
+
index stays empty
|
|
143
149
|
- Attachment file not found on disk → listed as `*(not available)*` in markdown
|
|
144
150
|
- Attachment copy fails (permissions, disk full) → listed as `*(not available)*`
|
|
145
151
|
- Filename collision across messages → prefixed with `{message_id}_`
|
|
@@ -25,8 +25,8 @@ Typical path: `~/Library/Mail/V10/MailData/Envelope Index`
|
|
|
25
25
|
| `flagged` | INTEGER | 1 = flagged |
|
|
26
26
|
|
|
27
27
|
**IMPORTANT:** `date_received` stores **Unix timestamps** (seconds since
|
|
28
|
-
1970-01-01 UTC)
|
|
29
|
-
apply Core Data conversion.
|
|
28
|
+
1970-01-01 UTC). It does NOT store Core Data timestamps, which use the
|
|
29
|
+
2001-01-01 epoch. Do NOT apply Core Data conversion.
|
|
30
30
|
|
|
31
31
|
## addresses (sender and recipient addresses)
|
|
32
32
|
|
|
@@ -36,8 +36,8 @@ apply Core Data conversion.
|
|
|
36
36
|
| `address` | TEXT | Email address |
|
|
37
37
|
| `comment` | TEXT | Display name (e.g., `"Chen, Sarah"`) |
|
|
38
38
|
|
|
39
|
-
**IMPORTANT:** The display name is in `comment
|
|
40
|
-
column.
|
|
39
|
+
**IMPORTANT:** The display name is in `comment`. There is no `name` column and
|
|
40
|
+
no `display_name` column.
|
|
41
41
|
|
|
42
42
|
## subjects
|
|
43
43
|
|
|
@@ -46,8 +46,8 @@ column.
|
|
|
46
46
|
| `ROWID` | INTEGER | Primary key |
|
|
47
47
|
| `subject` | TEXT | Base subject text |
|
|
48
48
|
|
|
49
|
-
Note: `subject_prefix` (Re:, Fwd:, etc.)
|
|
50
|
-
|
|
49
|
+
Note: the `messages` table stores `subject_prefix` (Re:, Fwd:, etc.) directly.
|
|
50
|
+
This table does not store it.
|
|
51
51
|
|
|
52
52
|
## recipients (To/Cc/Bcc per message)
|
|
53
53
|
|
|
@@ -59,8 +59,8 @@ directly, not here.
|
|
|
59
59
|
| `type` | INTEGER | 0 = To, 1 = Cc, 2 = Bcc |
|
|
60
60
|
| `position` | INTEGER | Order within the type group |
|
|
61
61
|
|
|
62
|
-
**IMPORTANT:**
|
|
63
|
-
`address_id
|
|
62
|
+
**IMPORTANT:** The column names are `message` and `address`. They are not
|
|
63
|
+
`message_id` or `address_id`.
|
|
64
64
|
|
|
65
65
|
## summaries (Apple Intelligence email summaries)
|
|
66
66
|
|
|
@@ -99,8 +99,8 @@ Use case-insensitive `LIKE` patterns to match both:
|
|
|
99
99
|
**Constraints:** `UNIQUE(message, attachment_id)` — each attachment within a
|
|
100
100
|
message has a unique identifier.
|
|
101
101
|
|
|
102
|
-
**IMPORTANT:**
|
|
103
|
-
|
|
102
|
+
**IMPORTANT:** The column is `message`. It is not `message_id`. This matches the
|
|
103
|
+
convention of the `recipients` table.
|
|
104
104
|
|
|
105
105
|
### Filesystem mapping
|
|
106
106
|
|
|
@@ -3,11 +3,11 @@
|
|
|
3
3
|
* Parse a macOS Mail .emlx or .partial.emlx file and output the plain text body.
|
|
4
4
|
*
|
|
5
5
|
* The .emlx format is: first line = byte count, then RFC822 message, then Apple
|
|
6
|
-
* plist. This script reads the RFC822 portion
|
|
7
|
-
* text/plain
|
|
8
|
-
*
|
|
6
|
+
* plist. This script reads the RFC822 portion. It walks the MIME parts to find
|
|
7
|
+
* text/plain. It prints that part to stdout. If the email is HTML-only, the
|
|
8
|
+
* script strips the tags and decodes the entities instead.
|
|
9
9
|
*
|
|
10
|
-
*
|
|
10
|
+
* It also exports `parseEmlx()` and `extractBody()` for sync-apple-mail.
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
13
|
if (process.argv.includes("-h") || process.argv.includes("--help")) {
|
|
@@ -16,7 +16,7 @@ if (process.argv.includes("-h") || process.argv.includes("--help")) {
|
|
|
16
16
|
Usage: node scripts/parse-emlx.mjs <path-to-emlx-file> [-h|--help]
|
|
17
17
|
|
|
18
18
|
Parses a macOS Mail .emlx or .partial.emlx file and prints the plain text
|
|
19
|
-
body to stdout.
|
|
19
|
+
body to stdout. For an HTML-only email, it strips the HTML tags instead.`);
|
|
20
20
|
process.exit(0);
|
|
21
21
|
}
|
|
22
22
|
|
|
@@ -127,7 +127,7 @@ function parseHeaders(raw) {
|
|
|
127
127
|
continue;
|
|
128
128
|
}
|
|
129
129
|
|
|
130
|
-
// New header
|
|
130
|
+
// New header. Save the previous one.
|
|
131
131
|
pushHeader(headers, currentName, currentValue);
|
|
132
132
|
|
|
133
133
|
const colonIdx = line.indexOf(":");
|
|
@@ -141,7 +141,7 @@ function parseHeaders(raw) {
|
|
|
141
141
|
}
|
|
142
142
|
|
|
143
143
|
/**
|
|
144
|
-
* Parse Content-Type header value.
|
|
144
|
+
* Parse a Content-Type header value.
|
|
145
145
|
* @param {string} value - e.g. 'text/plain; charset="utf-8"; boundary="abc"'
|
|
146
146
|
* @returns {{ type: string, params: Record<string, string> }}
|
|
147
147
|
*/
|
|
@@ -217,7 +217,7 @@ function decodePayload(data, encoding) {
|
|
|
217
217
|
}
|
|
218
218
|
|
|
219
219
|
/**
|
|
220
|
-
* Decode text from a buffer
|
|
220
|
+
* Decode text from a buffer with the given charset.
|
|
221
221
|
* @param {Buffer} data
|
|
222
222
|
* @param {string} [charset]
|
|
223
223
|
* @returns {string}
|
|
@@ -342,7 +342,7 @@ export function extractBody(raw) {
|
|
|
342
342
|
*/
|
|
343
343
|
export function parseEmlx(filePath) {
|
|
344
344
|
const data = readFileSync(filePath);
|
|
345
|
-
//
|
|
345
|
+
// The first line is the byte count
|
|
346
346
|
const newline = data.indexOf(0x0a);
|
|
347
347
|
const byteCount = parseInt(data.subarray(0, newline).toString("ascii"), 10);
|
|
348
348
|
const raw = data.subarray(newline + 1, newline + 1 + byteCount);
|
|
@@ -11,9 +11,10 @@ import { basename, join, resolve, sep } from "node:path";
|
|
|
11
11
|
import { homedir } from "node:os";
|
|
12
12
|
import { globSync } from "node:fs";
|
|
13
13
|
|
|
14
|
-
// node:sqlite
|
|
15
|
-
//
|
|
16
|
-
// site is openDb below
|
|
14
|
+
// This module loads node:sqlite lazily through createRequire. Test environments
|
|
15
|
+
// (bun, older node) that lack the built-in can then import this module. The
|
|
16
|
+
// only call site is openDb below. Production runs on Node 22+, where
|
|
17
|
+
// node:sqlite resolves.
|
|
17
18
|
const requireModule = createRequire(import.meta.url);
|
|
18
19
|
|
|
19
20
|
const HOME = homedir();
|
|
@@ -31,13 +32,15 @@ export function findDb() {
|
|
|
31
32
|
.sort()
|
|
32
33
|
.reverse();
|
|
33
34
|
if (paths.length === 0) {
|
|
34
|
-
console.error(
|
|
35
|
+
console.error(
|
|
36
|
+
"Error: Apple Mail database not found. Check that Mail is configured.",
|
|
37
|
+
);
|
|
35
38
|
process.exit(1);
|
|
36
39
|
}
|
|
37
40
|
return paths[0];
|
|
38
41
|
}
|
|
39
42
|
|
|
40
|
-
/** Open a read-only SQLite connection
|
|
43
|
+
/** Open a read-only SQLite connection. Retry once if the database is locked. */
|
|
41
44
|
export function openDb(dbPath) {
|
|
42
45
|
const { DatabaseSync } = requireModule("node:sqlite");
|
|
43
46
|
try {
|
|
@@ -51,7 +54,7 @@ export function openDb(dbPath) {
|
|
|
51
54
|
}
|
|
52
55
|
}
|
|
53
56
|
|
|
54
|
-
/** Execute a SQL query and return all rows
|
|
57
|
+
/** Execute a SQL query and return all rows. Log the error and return an empty array on failure. */
|
|
55
58
|
export function query(db, sql) {
|
|
56
59
|
try {
|
|
57
60
|
return db.prepare(sql).all();
|
|
@@ -61,7 +64,7 @@ export function query(db, sql) {
|
|
|
61
64
|
}
|
|
62
65
|
}
|
|
63
66
|
|
|
64
|
-
/** Load the last sync timestamp from disk
|
|
67
|
+
/** Load the last sync timestamp from disk. Use daysBack days ago on the first run. */
|
|
65
68
|
export function loadLastSync(daysBack = 30) {
|
|
66
69
|
try {
|
|
67
70
|
const iso = readFileSync(STATE_FILE, "utf-8").trim();
|
|
@@ -84,7 +87,7 @@ export function saveSyncState(lastRowid = null) {
|
|
|
84
87
|
}
|
|
85
88
|
}
|
|
86
89
|
|
|
87
|
-
/** Load the last-seen message ROWID from disk
|
|
90
|
+
/** Load the last-seen message ROWID from disk. Return 0 on the first run. */
|
|
88
91
|
export function loadLastRowid() {
|
|
89
92
|
try {
|
|
90
93
|
const val = readFileSync(ROWID_STATE_FILE, "utf-8").trim();
|
|
@@ -108,7 +111,7 @@ export function unixToReadable(ts) {
|
|
|
108
111
|
}
|
|
109
112
|
}
|
|
110
113
|
|
|
111
|
-
/** Detect whether the messages table uses conversation_id or thread_id
|
|
114
|
+
/** Detect whether the messages table uses conversation_id or thread_id as the thread column. */
|
|
112
115
|
export function discoverThreadColumn(db) {
|
|
113
116
|
const rows = query(db, "PRAGMA table_info(messages);");
|
|
114
117
|
const columns = new Set(rows.map((r) => r.name));
|
|
@@ -137,7 +140,7 @@ export function findChangedThreads(db, threadCol, sinceTs, lastRowid) {
|
|
|
137
140
|
);
|
|
138
141
|
}
|
|
139
142
|
|
|
140
|
-
/** Fetch all messages in a thread ordered by date,
|
|
143
|
+
/** Fetch all messages in a thread ordered by date, with sender and summary metadata. */
|
|
141
144
|
export function fetchThreadMessages(db, threadCol, tid) {
|
|
142
145
|
return query(
|
|
143
146
|
db,
|
|
@@ -217,13 +220,13 @@ export function fetchAttachments(db, messageIds) {
|
|
|
217
220
|
}
|
|
218
221
|
|
|
219
222
|
const FALLBACK_ATTACHMENT_NAME = "unnamed";
|
|
220
|
-
// biome-ignore lint/suspicious/noControlCharactersInRegex: intentional
|
|
223
|
+
// biome-ignore lint/suspicious/noControlCharactersInRegex: intentional. It strips ASCII control bytes from attacker-controlled filenames.
|
|
221
224
|
const CONTROL_CHARS_RE = /[\x00-\x1f\x7f]/g;
|
|
222
225
|
|
|
223
226
|
/**
|
|
224
227
|
* Coerce an arbitrary `attachments.name` value into a single, non-empty
|
|
225
228
|
* basename safe to join under a per-thread destDir. Strips path separators
|
|
226
|
-
* (POSIX and win32)
|
|
229
|
+
* (POSIX and win32). Strips ASCII control bytes. Then takes the last
|
|
227
230
|
* non-empty/non-dot segment. Returns `"unnamed"` for any input that
|
|
228
231
|
* collapses to empty, `.`, or `..`. Never throws.
|
|
229
232
|
*/
|
|
@@ -341,7 +344,7 @@ function copySingleAttachment(
|
|
|
341
344
|
}
|
|
342
345
|
}
|
|
343
346
|
|
|
344
|
-
/** Copy all attachments for a thread's messages into the cache directory
|
|
347
|
+
/** Copy all attachments for a thread's messages into the cache directory and deduplicate the filenames. `attachmentsDir` defaults to the module-level `ATTACHMENTS_DIR`. Tests inject a temp directory. */
|
|
345
348
|
export function copyThreadAttachments(
|
|
346
349
|
threadId,
|
|
347
350
|
messages,
|
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
* Sync Apple Mail threads to ~/.cache/fit/outpost/apple_mail/ as markdown.
|
|
4
4
|
*
|
|
5
5
|
* Queries the macOS Mail Envelope Index SQLite database for threads with new
|
|
6
|
-
* messages since the last sync. Writes one markdown file per thread
|
|
7
|
-
* sender, recipients, date, body text (parsed from .emlx files), and
|
|
8
|
-
* links.
|
|
6
|
+
* messages since the last sync. Writes one markdown file per thread. Each file
|
|
7
|
+
* holds the sender, recipients, date, body text (parsed from .emlx files), and
|
|
8
|
+
* attachment links. Copies the attachments into a per-thread subdirectory.
|
|
9
9
|
*
|
|
10
10
|
* Requires macOS with Mail app configured and Full Disk Access granted.
|
|
11
11
|
*/
|
|
@@ -1,21 +1,24 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: sync-teams
|
|
3
|
-
description: Sync recent Microsoft Teams chat messages into ~/.cache/fit/outpost/teams_chat/ as markdown files
|
|
3
|
+
description: Sync recent Microsoft Teams chat messages into ~/.cache/fit/outpost/teams_chat/ as markdown files. The skill reads the Teams IndexedDB cache from disk. Use on a schedule or when the user asks to sync their Teams chats. Requires macOS with the Teams desktop app installed.
|
|
4
4
|
compatibility: Requires macOS with Microsoft Teams desktop app (com.microsoft.teams2) installed
|
|
5
5
|
---
|
|
6
6
|
|
|
7
7
|
# Sync Teams
|
|
8
8
|
|
|
9
|
+
Write tier: none (cache only)
|
|
10
|
+
Frontmatter: none
|
|
11
|
+
|
|
9
12
|
Sync recent Microsoft Teams chat messages into
|
|
10
|
-
`~/.cache/fit/outpost/teams_chat/` as markdown files. This is an automated
|
|
11
|
-
|
|
13
|
+
`~/.cache/fit/outpost/teams_chat/` as markdown files. This is an automated skill
|
|
14
|
+
in the data pipeline. It ingests chat data that other skills (like
|
|
12
15
|
`extract-entities`) consume downstream.
|
|
13
16
|
|
|
14
|
-
This skill reads the Teams IndexedDB cache directly from disk
|
|
15
|
-
automation, no API tokens, no network access
|
|
16
|
-
|
|
17
|
-
IndexedDB at a known location. This skill parses those files,
|
|
18
|
-
V8-encoded records, and writes markdown.
|
|
17
|
+
This skill reads the Teams IndexedDB cache directly from disk. It needs no
|
|
18
|
+
browser automation, no API tokens, and no network access. The Teams desktop app
|
|
19
|
+
uses Edge WebView2. The app stores conversations and messages in a
|
|
20
|
+
LevelDB-backed IndexedDB at a known location. This skill parses those files,
|
|
21
|
+
deserializes the V8-encoded records, and writes markdown.
|
|
19
22
|
|
|
20
23
|
## Trigger
|
|
21
24
|
|
|
@@ -59,7 +62,7 @@ The script:
|
|
|
59
62
|
1. Reads all LevelDB `.ldb` (SSTable) and `.log` (write-ahead log) files from
|
|
60
63
|
the Teams IndexedDB directory
|
|
61
64
|
2. Decompresses Snappy-compressed blocks and deserializes V8-encoded values
|
|
62
|
-
|
|
65
|
+
with Node's built-in `v8.deserialize()`
|
|
63
66
|
3. Extracts conversation records (with member lists, topics, chat type) and
|
|
64
67
|
message records (with sender names, HTML content, timestamps)
|
|
65
68
|
4. Groups messages by conversation, filters by date window, and converts HTML
|
|
@@ -69,7 +72,7 @@ The script:
|
|
|
69
72
|
|
|
70
73
|
### Architecture
|
|
71
74
|
|
|
72
|
-
Three modules
|
|
75
|
+
Three modules follow the same pattern as `sync-apple-mail`:
|
|
73
76
|
|
|
74
77
|
| Module | Purpose |
|
|
75
78
|
| ---------------------------- | --------------------------------------------------------------------------------------------------------------------------- |
|
|
@@ -87,17 +90,17 @@ data in LevelDB (the same way Chrome does). The key databases are:
|
|
|
87
90
|
- **replychain-manager** — stores actual messages: sender display name, HTML
|
|
88
91
|
content, timestamps, reactions, edit status
|
|
89
92
|
|
|
90
|
-
|
|
91
|
-
|
|
93
|
+
LevelDB is an append-only format. So you can read the files while Teams runs.
|
|
94
|
+
Newer `.ldb` files supersede older ones for the same records.
|
|
92
95
|
|
|
93
96
|
### Name Resolution
|
|
94
97
|
|
|
95
|
-
Teams conversation records
|
|
96
|
-
identifiers.
|
|
98
|
+
Teams conversation records do not store human-readable member names. They store
|
|
99
|
+
orgid identifiers only. Resolve display names from:
|
|
97
100
|
|
|
98
101
|
1. **Conversation topic** (for named group chats)
|
|
99
|
-
2. **Message sender names** (`imDisplayName` field) —
|
|
100
|
-
|
|
102
|
+
2. **Message sender names** (`imDisplayName` field) — a 1:1 chat takes the name
|
|
103
|
+
of the other participant(s)
|
|
101
104
|
|
|
102
105
|
## Output Format
|
|
103
106
|
|
|
@@ -144,14 +147,13 @@ Key conventions:
|
|
|
144
147
|
|
|
145
148
|
- Messages in **chronological order** (oldest first)
|
|
146
149
|
- **Normalize names** from Teams format ("Last, First") to "First Last"
|
|
147
|
-
- **Platform** line distinguishes Teams from email
|
|
148
|
-
- **Plain text only** —
|
|
149
|
-
- **
|
|
150
|
-
the markdown.
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
there first.
|
|
150
|
+
- The **Platform** line distinguishes Teams from email downstream
|
|
151
|
+
- **Plain text only** — the script strips HTML. Mentions stay as plain text
|
|
152
|
+
- **The script does not extract attachments** — it drops files and images from
|
|
153
|
+
the markdown. SharePoint or OneDrive hosts them. The local cache does not. But
|
|
154
|
+
the user **often downloads them manually**, so an attachment usually exists
|
|
155
|
+
under `~/Downloads/` with the **same file name** Teams shows. When a message
|
|
156
|
+
references an attachment and you need its contents, look there first.
|
|
155
157
|
- Skip system messages (calls, member adds/removes, topic changes)
|
|
156
158
|
|
|
157
159
|
## Error Handling
|
|
@@ -161,29 +163,30 @@ Key conventions:
|
|
|
161
163
|
- LevelDB file parse error → skip that file, continue with others
|
|
162
164
|
- V8 deserialization failure → skip that record, continue
|
|
163
165
|
- Snappy decompression failure → skip that block, continue
|
|
164
|
-
- Empty chat (no messages in window) → skip,
|
|
166
|
+
- Empty chat (no messages in window) → skip, do not write a file
|
|
165
167
|
- Always update sync state, even on partial success
|
|
166
168
|
|
|
167
169
|
## Constraints
|
|
168
170
|
|
|
169
|
-
- **Read-only.**
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
- **
|
|
174
|
-
|
|
171
|
+
- **Read-only.** The script never writes to the Teams IndexedDB or sends
|
|
172
|
+
messages.
|
|
173
|
+
- **Cache-dependent.** Only the conversations Teams caches locally are
|
|
174
|
+
available. This covers recently viewed chats. It does not cover full history.
|
|
175
|
+
- **Both 1:1 and group chats.** The script syncs both and excludes channels.
|
|
176
|
+
- **No message limit per chat** — the output holds every cached message within
|
|
177
|
+
the `--days` window.
|
|
175
178
|
|
|
176
179
|
## Limitations
|
|
177
180
|
|
|
178
|
-
- The IndexedDB is a **cache
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
-
|
|
182
|
-
|
|
183
|
-
- Some V8-serialized records (~17% in
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
- **
|
|
187
|
-
message text
|
|
188
|
-
user
|
|
189
|
-
`~/Downloads/`. Check there before
|
|
181
|
+
- The IndexedDB is a **cache**. It is not an archive. Only the conversations the
|
|
182
|
+
user opened recently in Teams have cached message data. An older conversation
|
|
183
|
+
the user never opened may have conversation metadata but no messages.
|
|
184
|
+
- If you clear the Teams cache (a common troubleshooting step), all local data
|
|
185
|
+
goes away until Teams rebuilds it from the server.
|
|
186
|
+
- Some V8-serialized records (~17% in tests) use formats that `v8.deserialize()`
|
|
187
|
+
cannot decode. The script skips them silently. They are typically IndexedDB
|
|
188
|
+
metadata. They are not conversation or message records.
|
|
189
|
+
- **The script never syncs attachments (files/images) into the markdown** — it
|
|
190
|
+
captures only the message text. The binaries live on SharePoint/OneDrive. But
|
|
191
|
+
the user downloads them often, so the same-named file is usually already in
|
|
192
|
+
`~/Downloads/`. Check there before you fetch from SharePoint.
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
*
|
|
5
5
|
* Chromium stores IndexedDB data in LevelDB with a specific key encoding
|
|
6
6
|
* (database ID, object store ID) and V8-serialized values wrapped in a Blink
|
|
7
|
-
* envelope. This module
|
|
7
|
+
* envelope. This module parses the keys and deserializes the values.
|
|
8
8
|
*
|
|
9
9
|
* Exports: readIndexedDb(dir) → { conversations: Map, messages: Map }
|
|
10
10
|
*/
|
|
@@ -32,18 +32,18 @@ function readIdbVarint(buf, offset) {
|
|
|
32
32
|
}
|
|
33
33
|
|
|
34
34
|
// Highest V8 serialization wire-format version Node's bundled v8.deserialize
|
|
35
|
-
// accepts. Newer Teams/WebView2 builds write version 16
|
|
36
|
-
// outright even though the payload itself is wire-compatible. We patch the
|
|
37
|
-
// version byte down to this value before
|
|
38
|
-
//
|
|
35
|
+
// accepts. Newer Teams/WebView2 builds write version 16. Node rejects it
|
|
36
|
+
// outright, even though the payload itself is wire-compatible. We patch the
|
|
37
|
+
// version byte down to this value before we deserialize. Bump this value when
|
|
38
|
+
// Node's V8 emits or accepts a higher version natively.
|
|
39
39
|
const V8_MAX_SUPPORTED_VERSION = 15;
|
|
40
40
|
|
|
41
41
|
// Plausible V8 top-level value tags that immediately follow the
|
|
42
|
-
// [0xFF <version>] header.
|
|
43
|
-
// Blink envelope
|
|
44
|
-
// a 0xFE trailer that shifts the payload further
|
|
45
|
-
//
|
|
46
|
-
//
|
|
42
|
+
// [0xFF <version>] header. The reader uses them to locate where the real V8
|
|
43
|
+
// payload starts inside the Blink envelope. It does not depend on a fixed byte
|
|
44
|
+
// offset (newer envelopes carry a 0xFE trailer that shifts the payload further
|
|
45
|
+
// in). We only ever ACT on a candidate when we try to deserialize it. That
|
|
46
|
+
// call validates the candidate, so the code skips a stray match.
|
|
47
47
|
const V8_TOP_LEVEL_TAGS = new Set([
|
|
48
48
|
0x6f, // 'o' begin JS object
|
|
49
49
|
0x22, // '"' one-byte string
|
|
@@ -62,16 +62,17 @@ const V8_TOP_LEVEL_TAGS = new Set([
|
|
|
62
62
|
0x30, // '0' null
|
|
63
63
|
]);
|
|
64
64
|
|
|
65
|
-
// Only the Blink envelope precedes the V8 payload
|
|
66
|
-
//
|
|
67
|
-
// cheap
|
|
65
|
+
// Only the Blink envelope precedes the V8 payload. The envelope is always
|
|
66
|
+
// small. A scan over a generous prefix keeps non-message records (which never
|
|
67
|
+
// decode) cheap. It also covers every real envelope/trailer layout comfortably.
|
|
68
68
|
const V8_START_SCAN_LIMIT = 256;
|
|
69
69
|
|
|
70
70
|
/**
|
|
71
|
-
* Deserialize the V8 payload
|
|
72
|
-
*
|
|
73
|
-
* with the version patched down. The wire format is
|
|
74
|
-
* supported version reads the newer payload
|
|
71
|
+
* Deserialize the V8 payload that starts at `off`. Tries the bytes as-is
|
|
72
|
+
* first. For a record whose version byte is newer than Node supports, it then
|
|
73
|
+
* retries with the version patched down. The wire format is
|
|
74
|
+
* backward-compatible, so a supported version reads the newer payload
|
|
75
|
+
* correctly.
|
|
75
76
|
*/
|
|
76
77
|
function deserializeAt(rawValue, off) {
|
|
77
78
|
try {
|
|
@@ -99,9 +100,9 @@ function deserializeAt(rawValue, off) {
|
|
|
99
100
|
* Try to deserialize a Chromium IndexedDB value.
|
|
100
101
|
*
|
|
101
102
|
* Values have a Blink envelope (and, in newer WebView2 builds, a 0xFE trailer)
|
|
102
|
-
* before the V8 payload.
|
|
103
|
-
*
|
|
104
|
-
*
|
|
103
|
+
* before the V8 payload. Scan for a [0xFF <version> <top-level tag>] header to
|
|
104
|
+
* locate the payload. Then decode it. For a record written with a V8 wire
|
|
105
|
+
* version newer than Node accepts, patch the version byte down first.
|
|
105
106
|
*/
|
|
106
107
|
function tryDeserialize(rawValue) {
|
|
107
108
|
if (!rawValue || rawValue.length < 4) return null;
|
|
@@ -141,7 +142,7 @@ function _parseKeyPrefix(key) {
|
|
|
141
142
|
*/
|
|
142
143
|
export function readIndexedDb(dir) {
|
|
143
144
|
// Use Maps so later entries (from newer .ldb files) overwrite older ones.
|
|
144
|
-
// LevelDB reads files in ascending order
|
|
145
|
+
// LevelDB reads files in ascending order. Newer compactions have higher
|
|
145
146
|
// numbers, so the last write for a given key is the most current.
|
|
146
147
|
const convMap = new Map();
|
|
147
148
|
const msgMap = new Map();
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
/**
|
|
3
3
|
* Read LevelDB SSTable (.ldb) and write-ahead log (.log) files.
|
|
4
4
|
*
|
|
5
|
-
* Parses the binary SSTable format directly
|
|
6
|
-
* Handles Snappy-compressed blocks
|
|
5
|
+
* Parses the binary SSTable format directly. It needs no LevelDB library.
|
|
6
|
+
* Handles Snappy-compressed blocks with snappyjs (pure JS, zero native deps).
|
|
7
7
|
*
|
|
8
8
|
* Exports a single generator function: readAllEntries(directory)
|
|
9
9
|
*/
|
|
@@ -146,7 +146,7 @@ const LOG_BLOCK_SIZE = 32768;
|
|
|
146
146
|
const LOG_HEADER_SIZE = 7; // checksum(4) + length(2) + type(1)
|
|
147
147
|
|
|
148
148
|
/**
|
|
149
|
-
* Parse records from a single WAL block
|
|
149
|
+
* Parse records from a single WAL block. Yield { payload, type } for each one.
|
|
150
150
|
*/
|
|
151
151
|
function* parseLogBlock(fileData, blockStart, blockEnd) {
|
|
152
152
|
let offset = blockStart;
|
|
@@ -244,7 +244,7 @@ function* parseWriteBatchEntries(batchData) {
|
|
|
244
244
|
|
|
245
245
|
yield { key, value };
|
|
246
246
|
} else if (tag === 0) {
|
|
247
|
-
// Delete
|
|
247
|
+
// Delete. Skip the key.
|
|
248
248
|
const keyLen = readVarint(batchData, pos);
|
|
249
249
|
pos += keyLen.bytesRead;
|
|
250
250
|
pos += keyLen.value;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
/**
|
|
3
3
|
* Sync Microsoft Teams chat messages to ~/.cache/fit/outpost/teams_chat/ as
|
|
4
|
-
* markdown files. Reads the Teams IndexedDB cache (LevelDB on disk) directly
|
|
5
|
-
* no browser automation, no API tokens, no network access
|
|
4
|
+
* markdown files. Reads the Teams IndexedDB cache (LevelDB on disk) directly.
|
|
5
|
+
* It needs no browser automation, no API tokens, and no network access.
|
|
6
6
|
*
|
|
7
7
|
* Requires macOS with the Microsoft Teams desktop app installed.
|
|
8
8
|
* Requires snappyjs: npm install snappyjs
|
|
@@ -272,7 +272,7 @@ function loadUserIdentity() {
|
|
|
272
272
|
const nameMatch = identityMd.match(/\*\*Name:\*\*\s*(.+)/);
|
|
273
273
|
if (nameMatch) return nameMatch[1].trim();
|
|
274
274
|
} catch {
|
|
275
|
-
// identity cache not found or not readable
|
|
275
|
+
// identity cache not found or not readable. Run the identify-user skill
|
|
276
276
|
}
|
|
277
277
|
return "";
|
|
278
278
|
}
|
|
@@ -326,7 +326,7 @@ function main() {
|
|
|
326
326
|
|
|
327
327
|
if (!existsSync(TEAMS_IDB_DIR)) {
|
|
328
328
|
console.error(
|
|
329
|
-
"Error: Teams IndexedDB not found.
|
|
329
|
+
"Error: Teams IndexedDB not found. Check that the Teams desktop app is installed.",
|
|
330
330
|
);
|
|
331
331
|
console.error(`Expected: ${TEAMS_IDB_DIR}`);
|
|
332
332
|
process.exit(1);
|