@chronicle.app/slack 0.4.0 → 0.5.0

package/README.md

@@ -45,7 +45,7 @@ chronicle extract slack --input ./my-slack-export --since 2024-01-01
 
 ## Configuration
 
-- `--input` (required): path to the slackdump export directory
+- `--input` (required): the slackdump export, as a folder or zip
 - `--attachments`: include file attachments (default: false)
 - `--since`: only extract messages on or after this date
 - `--until`: only extract messages on or before this date
@@ -55,7 +55,7 @@ chronicle extract slack --input ./my-slack-export --since 2024-01-01
 
 This plugin maps Slack data onto the following Chronicle types:
 
-- **Messages** → `MessageAction` whose `object` is a `Message` (`source: slack`, `body`, `author`, `recipient`)
+- **Messages** → `MessageAction` whose `result` is a `Message` (`source: slack`, `body`, `author`, `recipient`)
 - **Users** → `Agent` (resolved to a `handle` and `name` from `users.json`, group-DM metadata, and message profiles)
 - **File attachments** → `ImageObject` / `AudioObject` / `VideoObject` / `DocumentObject` (by MIME type), with `contentPath` to the locally exported file when available
 

package/dist/UserResolver.d.ts

@@ -1,3 +1,4 @@
+import type { ExportFiles } from '@chronicle.app/etl';
 import type { ResolvedUser } from './types/slack.js';
 /**
  * Resolves Slack user IDs to names/handles by combining multiple data sources:
@@ -9,7 +10,7 @@ export declare class UserResolver {
     private users;
     get(userId: string): ResolvedUser | undefined;
     getOrStub(userId: string, teamId?: string): ResolvedUser;
-    resolve(basePath: string): Promise<Map<string, ResolvedUser>>;
+    resolve(files: ExportFiles): Promise<Map<string, ResolvedUser>>;
     private loadUsersJson;
     private loadMpimCoOccurrence;
     private scanMessagesForProfiles;

package/dist/UserResolver.js

@@ -1,5 +1,3 @@
-import { readFile, readdir } from 'node:fs/promises';
-import { join } from 'node:path';
 /**
  * Resolves Slack user IDs to names/handles by combining multiple data sources:
  * 1. users.json - workspace members with full profiles
@@ -17,18 +15,18 @@ export class UserResolver {
             return existing;
         return { id: userId, teamId };
     }
-    async resolve(basePath) {
+    async resolve(files) {
         // Phase 1: Load users.json (most complete source)
-        await this.loadUsersJson(basePath);
+        await this.loadUsersJson(files);
         // Phase 2: Co-occurrence from mpims.json purpose fields
-        await this.loadMpimCoOccurrence(basePath);
+        await this.loadMpimCoOccurrence(files);
         // Phase 3: Scan messages for user_profile fields
-        await this.scanMessagesForProfiles(basePath);
+        await this.scanMessagesForProfiles(files);
         return this.users;
     }
-    async loadUsersJson(basePath) {
+    async loadUsersJson(files) {
         try {
-            const raw = await readFile(join(basePath, 'users.json'), 'utf-8');
+            const raw = await files.readText('users.json');
             const users = JSON.parse(raw);
             for (const u of users) {
                 const name = u.profile?.display_name || u.profile?.real_name || u.real_name || undefined;
@@ -44,9 +42,9 @@ export class UserResolver {
             // users.json may not exist
         }
     }
-    async loadMpimCoOccurrence(basePath) {
+    async loadMpimCoOccurrence(files) {
         try {
-            const raw = await readFile(join(basePath, 'mpims.json'), 'utf-8');
+            const raw = await files.readText('mpims.json');
             const mpims = JSON.parse(raw);
             // Iterate until stable: match member IDs to @usernames in purpose field
             for (const mpim of mpims) {
@@ -85,19 +83,18 @@ export class UserResolver {
             // mpims.json may not exist
         }
     }
-    async scanMessagesForProfiles(basePath) {
+    async scanMessagesForProfiles(files) {
         try {
-            const entries = await readdir(basePath, { withFileTypes: true });
-            const dirs = entries.filter(e => e.isDirectory() && e.name !== 'attachments');
+            const dirs = (await files.children(''))
+                .filter(entry => entry.endsWith('/') && entry !== 'attachments/')
+                .map(entry => entry.slice(0, -1));
             for (const dir of dirs) {
-                const dirPath = join(basePath, dir.name);
-                const files = await readdir(dirPath).catch(() => []);
-                const jsonFiles = (typeof files[0] === 'string' ? files : []);
+                const jsonFiles = await files.children(dir);
                 for (const file of jsonFiles) {
                     if (!file.endsWith('.json'))
                         continue;
                     try {
-                        const raw = await readFile(join(dirPath, file), 'utf-8');
+                        const raw = await files.readText(`${dir}/${file}`);
                         const messages = JSON.parse(raw);
                         for (const msg of messages) {
                             if (!msg.user)

package/dist/connectors/SlackExtractor.d.ts

@@ -1,4 +1,4 @@
-import { Extractor, Record } from '@chronicle.app/etl';
+import { Extractor, Record, type KindSurvey, type Survey, type SurveyOptions } from '@chronicle.app/etl';
 import { z } from 'zod';
 import SlackTransformer from './SlackTransformer.js';
 export declare class SlackExtractor extends Extractor<typeof SlackExtractor> {
@@ -29,14 +29,30 @@ export declare class SlackExtractor extends Extractor<typeof SlackExtractor> {
         until?: Date | undefined;
         limit?: number | undefined;
     }>;
+    private files;
     private conversations;
     private userMap;
     private teamId;
     setup(): Promise<void>;
+    teardown(): Promise<void>;
     extract(): AsyncGenerator<Record>;
+    /**
+     * Each downloaded file's path on disk, by its id. slackdump writes a file as
+     * `<fileId>-<name>`, in the conversation's own `attachments/` folder or a
+     * top-level aggregate one; a sanitized name still matches by its id. A file
+     * slackdump didn't download has none.
+     */
+    private attachmentPaths;
     /** Yield a single conversation's messages newest-first, applying filters. */
     private extractConversation;
     determineCount(): Promise<number | null>;
+    /**
+     * The messages a run reads, when they were sent, and in how many
+     * conversations, from every day file in range: the export has no index.
+     */
+    survey(options?: SurveyOptions): Promise<{
+        messages: KindSurvey;
+    } & Survey>;
     private loadConversations;
     private tsToDate;
 }

package/dist/connectors/SlackExtractor.js

@@ -1,6 +1,4 @@
-import { Extractor } from '@chronicle.app/etl';
-import { readFile, readdir } from 'node:fs/promises';
-import { join } from 'node:path';
+import { Extractor, countMonth, openExport, } from '@chronicle.app/etl';
 import { z } from 'zod';
 import { UserResolver } from '../UserResolver.js';
 import SlackTransformer from './SlackTransformer.js';
@@ -13,18 +11,19 @@ export class SlackExtractor extends Extractor {
     static default = true;
     static defaultTransformer = SlackTransformer;
     static schema = Extractor.schema.extend({
-        input: z.string().describe('Path to slackdump export directory'),
+        input: z.string().describe('The slackdump export, as a folder or zip'),
         attachments: z.boolean().optional().default(false).describe('Include file attachments'),
     });
+    files;
     conversations = [];
     userMap = new Map();
     teamId;
     async setup() {
         const config = this.config;
-        const basePath = config.input;
+        this.files = await openExport(config.input, 'slackdump export');
         // Resolve users
         const resolver = new UserResolver();
-        this.userMap = await resolver.resolve(basePath);
+        this.userMap = await resolver.resolve(this.files);
         // Extract teamId from first user that has one
         for (const user of this.userMap.values()) {
             if (user.teamId) {
@@ -33,7 +32,11 @@ export class SlackExtractor extends Extractor {
             }
         }
         // Load conversation metadata
-        await this.loadConversations(basePath);
+        await this.loadConversations();
+    }
+    async teardown() {
+        await this.files?.close();
+        await super.teardown();
     }
     async *extract() {
         const config = this.config;
@@ -66,11 +69,21 @@ export class SlackExtractor extends Extractor {
             }
             const head = heads[maxIndex];
             const { msg, conversation } = head.value;
+            // Where each attached file's bytes are, for the transformer to point at.
+            const attachmentPaths = config.attachments && msg.files?.length
+                ? await this.attachmentPaths(msg.files, conversation.dirName)
+                : undefined;
             yield this.createRecord({
                 type: 'message',
                 message: msg,
                 includeAttachments: config.attachments,
-            }, { conversation, userMap, teamId: this.teamId, basePath });
+            }, {
+                conversation,
+                userMap,
+                teamId: this.teamId,
+                basePath,
+                ...(attachmentPaths && { attachmentPaths }),
+            });
             count++;
             // Advance the chosen stream; drop it once exhausted.
             const next = await head.iterator.next();
@@ -83,16 +96,30 @@ export class SlackExtractor extends Extractor {
             }
         }
     }
+    /**
+     * Each downloaded file's path on disk, by its id. slackdump writes a file as
+     * `<fileId>-<name>`, in the conversation's own `attachments/` folder or a
+     * top-level aggregate one; a sanitized name still matches by its id. A file
+     * slackdump didn't download has none.
+     */
+    async attachmentPaths(files, dirName) {
+        const paths = {};
+        for (const file of files) {
+            for (const dir of [`${dirName}/attachments`, 'attachments']) {
+                const match = (await this.files.children(dir)).find(entry => entry.startsWith(`${file.id}-`));
+                const path = match && (await this.files.localPath(`${dir}/${match}`));
+                if (path) {
+                    paths[file.id] = path;
+                    break;
+                }
+            }
+        }
+        return paths;
+    }
     /** Yield a single conversation's messages newest-first, applying filters. */
     async *extractConversation(conversation, config) {
-        const dirPath = join(config.input, conversation.dirName);
-        let files;
-        try {
-            files = await readdir(dirPath);
-        }
-        catch {
-            return;
-        }
+        const dirPath = conversation.dirName;
+        const files = await this.files.children(dirPath);
         // Dated JSON files, newest first.
         const jsonFiles = files
             .filter(f => /^\d{4}-\d{2}-\d{2}\.json$/.test(f))
@@ -108,7 +135,7 @@ export class SlackExtractor extends Extractor {
             }
             let messages;
             try {
-                const raw = await readFile(join(dirPath, file), 'utf-8');
+                const raw = await this.files.readText(`${dirPath}/${file}`);
                 messages = JSON.parse(raw);
             }
             catch {
@@ -136,37 +163,40 @@ export class SlackExtractor extends Extractor {
         }
     }
     async determineCount() {
+        const { count } = (await this.survey()).messages;
+        const limit = this.getEffectiveLimit();
+        return limit === null ? count : Math.min(count, limit);
+    }
+    /**
+     * The messages a run reads, when they were sent, and in how many
+     * conversations, from every day file in range: the export has no index.
+     */
+    async survey(options = {}) {
         const config = this.config;
-        const basePath = config.input;
-        let total = 0;
+        const messages = { count: 0, ...(options.byMonth && { byMonth: {} }) };
+        let conversations = 0;
         for (const conversation of this.conversations) {
-            const dirPath = join(basePath, conversation.dirName);
-            let files;
-            try {
-                files = await readdir(dirPath);
-            }
-            catch {
-                continue;
-            }
-            const jsonFiles = files.filter(f => /^\d{4}-\d{2}-\d{2}\.json$/.test(f));
-            for (const file of jsonFiles) {
-                try {
-                    const raw = await readFile(join(dirPath, file), 'utf-8');
-                    const messages = JSON.parse(raw);
-                    total += messages.filter(m => m.type === 'message' &&
-                        (!m.subtype || ['bot_message', 'thread_broadcast'].includes(m.subtype))).length;
-                }
-                catch {
-                    continue;
-                }
+            const before = messages.count;
+            for await (const { msg } of this.extractConversation(conversation, config)) {
+                const sent = this.tsToDate(msg.ts);
+                messages.count++;
+                if (!messages.earliest || sent < messages.earliest)
+                    messages.earliest = sent;
+                if (!messages.latest || sent > messages.latest)
+                    messages.latest = sent;
+                if (messages.byMonth)
+                    countMonth(messages.byMonth, sent);
             }
+            if (messages.count > before)
+                conversations++;
         }
-        return total;
+        messages.measures = [{ key: 'conversations', label: 'conversations', value: conversations }];
+        return { messages };
     }
-    async loadConversations(basePath) {
+    async loadConversations() {
         // Load channels
         try {
-            const raw = await readFile(join(basePath, 'channels.json'), 'utf-8');
+            const raw = await this.files.readText('channels.json');
             const channels = JSON.parse(raw);
             for (const ch of channels) {
                 this.conversations.push({
@@ -184,7 +214,7 @@ export class SlackExtractor extends Extractor {
         }
         // Load DMs
         try {
-            const raw = await readFile(join(basePath, 'dms.json'), 'utf-8');
+            const raw = await this.files.readText('dms.json');
             const dms = JSON.parse(raw);
             for (const dm of dms) {
                 this.conversations.push({
@@ -200,7 +230,7 @@ export class SlackExtractor extends Extractor {
         }
         // Load MPIMs
         try {
-            const raw = await readFile(join(basePath, 'mpims.json'), 'utf-8');
+            const raw = await this.files.readText('mpims.json');
             const mpims = JSON.parse(raw);
             for (const mpim of mpims) {
                 this.conversations.push({
@@ -215,9 +245,9 @@ export class SlackExtractor extends Extractor {
         catch {
             // mpims.json may not exist
         }
-        // Filter to only conversations that have directories on disk
-        const entries = await readdir(basePath, { withFileTypes: true }).catch(() => []);
-        const dirNames = new Set(entries.filter(e => e.isDirectory()).map(e => e.name));
+        // Filter to only conversations that have folders in the export
+        const entries = await this.files.children('');
+        const dirNames = new Set(entries.filter(e => e.endsWith('/')).map(e => e.slice(0, -1)));
         this.conversations = this.conversations.filter(c => dirNames.has(c.dirName));
     }
     tsToDate(ts) {

package/dist/connectors/SlackTransformer.js

@@ -34,7 +34,7 @@ export default class SlackTransformer extends ChronicleTransformer {
         }
         // Add attachments if enabled and present
         if (includeAttachments && msg.files && msg.files.length > 0) {
-            const attachments = this.buildAttachments(msg.files, record.context.basePath, conversation.dirName);
+            const attachments = this.buildAttachments(msg.files, record.context.basePath, conversation.dirName, record.context.attachmentPaths);
             if (attachments.length > 0) {
                 message.contains = attachments;
             }
@@ -46,7 +46,8 @@ export default class SlackTransformer extends ChronicleTransformer {
             source: 'slack',
             timestamp: new Date(Number.parseFloat(msg.ts) * 1000),
             agent: sender,
-            object: message,
+            // Sending makes the message, so it's the action's result.
+            result: message,
         };
         return [messageAction];
     }
@@ -113,15 +114,19 @@ export default class SlackTransformer extends ChronicleTransformer {
                 return [];
         }
     }
-    buildAttachments(files, basePath, dirName) {
+    buildAttachments(files, basePath, dirName, paths) {
         const attachments = [];
         for (const file of files) {
             // slackdump writes downloaded files to disk as `<fileId>-<name>`, in the
             // conversation's own `attachments/` dir (and a top-level aggregate one).
             // When the bytes are on disk we emit a contentPath so ingest captures
             // them; otherwise the node stays metadata-only (the url-gated Slack file
-            // would need an auth token to fetch).
-            const localPath = this.resolveAttachmentPath(basePath, dirName, file.id);
+            // would need an auth token to fetch). The extractor says where each is,
+            // a copy when the export is a zip; a record without that is looked up
+            // on disk under `basePath`.
+            const localPath = paths
+                ? paths[file.id]
+                : this.resolveAttachmentPath(basePath, dirName, file.id);
             const mediaObject = createMediaObject({
                 mimeType: file.mimetype,
                 source: 'slack',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chronicle.app/slack",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "files": [
@@ -22,9 +22,9 @@
   "type": "module",
   "description": "Chronicle plugin for Slack message history via slackdump export",
   "devDependencies": {
-    "@chronicle.app/etl": "0.4.0",
-    "@chronicle.app/schema": "0.4.0",
-    "@chronicle.app/tsconfig": "0.4.0",
+    "@chronicle.app/etl": "0.5.0",
+    "@chronicle.app/schema": "0.5.0",
+    "@chronicle.app/tsconfig": "0.5.0",
     "@types/node": "^22.10.1",
     "typescript": "^5.7.2"
   },
@@ -61,6 +61,14 @@
         "strategies": {
           "archive": {
             "delivery": "export",
+            "layouts": [
+              {
+                "files": [
+                  "channels.json",
+                  "users.json"
+                ]
+              }
+            ],
             "recordTypes": [
               "messages"
             ],