@chronicle.app/twitter 0.3.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Andrew Louis
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,76 @@
+# @chronicle.app/twitter
+
+Chronicle plugin for twitter
+
+## Installation
+
+```bash
+npm install @chronicle.app/twitter
+```
+
+## Usage
+
+### Basic extraction
+
+```bash
+chronicle extract twitter --input /path/to/twitter/export
+```
+
+### With date filtering
+
+```bash
+chronicle extract twitter \
+  --input /path/to/twitter/export \
+  --since 2024-01-01 \
+  --until 2024-12-31 \
+  --limit 1000
+```
+
+## Configuration
+
+The extractor supports the following options:
+
+- `--input` (required): Path to the Twitter export data directory
+- `--type`: Which record types to read: `tweets`, `likes`, `direct-messages`,
+  `group-direct-messages` (default: all of them)
+- `--include-group-dms`: Include group direct messages (default: true)
+- `--since`: Extract records from this date onwards
+- `--until`: Extract records up to this date
+- `--limit`: Maximum number of records to extract
+
+## Development
+
+```bash
+# Build the plugin
+npm run build
+
+# Run the tests
+npm test
+```
+
+## File Format
+
+The input is the unzipped directory of a Twitter/X data archive. The plugin
+reads these files from its `data/` folder, each a `window.YTD.<name>.part0 = [...]`
+script wrapping a JSON array:
+
+- `account.js`: the archive owner's account ID, username, and display name
+- `tweets.js`: your tweets
+- `like.js`: the tweets you liked
+- `direct-messages.js`: one-to-one direct messages
+- `direct-messages-group.js` and `direct-message-group-headers.js`: group direct
+  messages and their participants
+
+## Schema
+
+- Each tweet is a `PublishAction` whose object is a `Post`. The post's
+  `references` are the accounts it mentions and the links it contains.
+- Each like is a `LikeAction` on the liked `Post`.
+- Each direct message is a `MessageAction` whose object is a `Message`, sent to
+  the other participants of the conversation.
+
+The archive owner is an `Agent` linked to `@me`; other accounts are `Agent`s
+keyed by their Twitter user ID.
+
+Tests build a small synthetic archive in a temporary directory and never read
+the host's data or accounts.

package/dist/connectors/TwitterTransformer.d.ts

@@ -0,0 +1,14 @@
+import { ChronicleTransformer, Record } from '@chronicle.app/etl';
+import { ActionAndChildren } from '@chronicle.app/schema';
+export default class TwitterTransformer extends ChronicleTransformer {
+    transform(record: Record): Promise<ActionAndChildren[]>;
+    private buildTweetActions;
+    private buildLikeActions;
+    private buildDirectMessageActions;
+    private buildAccountUser;
+    private buildPersonFromUserId;
+    private extractMentionedUsersFromEntities;
+    private extractUrlReferencesFromEntities;
+    private buildGroupDirectMessageActions;
+}
+//# sourceMappingURL=TwitterTransformer.d.ts.map

package/dist/connectors/TwitterTransformer.js

@@ -0,0 +1,213 @@
+import { ChronicleTransformer } from '@chronicle.app/etl';
+export default class TwitterTransformer extends ChronicleTransformer {
+    async transform(record) {
+        const actions = [];
+        switch (record.extraction.recordType) {
+            case 'tweets':
+                actions.push(...(await this.buildTweetActions(record)));
+                break;
+            case 'likes':
+                actions.push(...this.buildLikeActions(record));
+                break;
+            case 'direct-messages':
+                actions.push(...this.buildDirectMessageActions(record));
+                break;
+            case 'group-direct-messages':
+                actions.push(...this.buildGroupDirectMessageActions(record));
+                break;
+        }
+        return actions;
+    }
+    async buildTweetActions(record) {
+        const actions = [];
+        const user = this.buildAccountUser(record);
+        const mentions = this.extractMentionedUsersFromEntities(record.data.entities, record.data.text);
+        const urls = this.extractUrlReferencesFromEntities(record.data.entities, record.data.text);
+        const references = [...mentions, ...urls];
+        const tweet = {
+            '@type': 'Post',
+            '@key': ['@type', 'source', 'sourceId'],
+            source: 'twitter',
+            sourceId: record.data.id,
+            body: record.data.text, // Keep original text with t.co URLs
+            author: [user],
+            ...(references.length > 0 && { references }),
+        };
+        const publishAction = {
+            '@type': 'PublishAction',
+            '@key': ['@type', 'source', 'sourceId'],
+            source: 'twitter',
+            sourceId: record.data.id,
+            timestamp: record.data.created_at,
+            agent: user,
+            object: tweet,
+        };
+        actions.push(publishAction);
+        return actions;
+    }
+    buildLikeActions(record) {
+        const actions = [];
+        const user = this.buildAccountUser(record);
+        const targetTweet = {
+            '@type': 'Post',
+            '@key': ['@type', 'source', 'sourceId'],
+            source: 'twitter',
+            sourceId: record.data.tweetId,
+            body: record.data.fullText,
+        };
+        const likeAction = {
+            '@type': 'LikeAction',
+            '@key': ['@type', 'source', 'sourceId'],
+            source: 'twitter',
+            sourceId: `${record.data.tweetId}-like`,
+            agent: user,
+            object: targetTweet,
+        };
+        actions.push(likeAction);
+        return actions;
+    }
+    buildDirectMessageActions(record) {
+        const actions = [];
+        const sender = this.buildPersonFromUserId(record.data.senderId);
+        const recipient = this.buildPersonFromUserId(record.data.recipientId);
+        const message = {
+            '@type': 'Message',
+            '@key': ['@type', 'source', 'sourceId'],
+            source: 'twitter',
+            sourceId: record.data.id,
+            body: record.data.text,
+            recipient: [recipient],
+            author: [sender],
+        };
+        // Media URLs would need to be handled separately or added to schema
+        const sendAction = {
+            '@type': 'MessageAction',
+            '@key': ['@type', 'source', 'sourceId'],
+            source: 'twitter',
+            sourceId: record.data.id,
+            timestamp: record.data.created_at,
+            agent: sender,
+            object: message,
+        };
+        actions.push(sendAction);
+        return actions;
+    }
+    buildAccountUser(record) {
+        // const strategy = record.context?.strategy || 'archive'; // Future use
+        const accountInfo = record.context?.accountInfo;
+        if (!accountInfo) {
+            throw new Error('Account information is required to build user agent. Ensure account.js is present in Twitter export.');
+        }
+        return {
+            '@type': 'Agent',
+            '@key': ['@type', 'source', 'sourceId'],
+            source: 'twitter',
+            sourceId: accountInfo.accountId,
+            handle: accountInfo.username,
+            name: accountInfo.accountDisplayName,
+            url: `https://twitter.com/${accountInfo.username}`,
+            sameAs: ['@me'],
+        };
+    }
+    buildPersonFromUserId(userId) {
+        return {
+            '@type': 'Agent',
+            '@key': ['@type', 'source', 'sourceId'],
+            source: 'twitter',
+            sourceId: userId,
+        };
+    }
+    extractMentionedUsersFromEntities(entities, text) {
+        const mentions = [];
+        // Primary: Use entities.user_mentions for structured data
+        if (entities?.user_mentions && entities.user_mentions.length > 0) {
+            for (const mention of entities.user_mentions) {
+                mentions.push({
+                    '@type': 'Agent',
+                    '@key': ['@type', 'source', 'sourceId'],
+                    source: 'twitter',
+                    sourceId: mention.id_str,
+                    handle: mention.screen_name,
+                    name: mention.name,
+                    url: `https://twitter.com/${mention.screen_name}`,
+                });
+            }
+        }
+        else {
+            // Fallback: Use regex parsing for edge cases where entities is empty
+            const mentionRegex = /@([a-zA-Z0-9_]+)/g;
+            let match;
+            while ((match = mentionRegex.exec(text)) !== null) {
+                const username = match[1];
+                mentions.push({
+                    '@type': 'Agent',
+                    '@key': ['@type', 'source', 'handle'],
+                    source: 'twitter',
+                    handle: username,
+                    url: `https://twitter.com/${username}`,
+                });
+            }
+        }
+        return mentions;
+    }
+    extractUrlReferencesFromEntities(entities, text) {
+        const urls = [];
+        // Primary: Use entities.urls for structured data
+        if (entities?.urls && entities.urls.length > 0) {
+            for (const urlEntity of entities.urls) {
+                // Use expanded URL if available, otherwise fall back to original
+                const finalUrl = urlEntity.expanded_url || urlEntity.url;
+                urls.push({
+                    '@type': 'Entity',
+                    '@key': ['@type', 'url'],
+                    url: finalUrl,
+                });
+            }
+        }
+        else {
+            // Fallback: Use regex parsing for edge cases where entities is empty
+            const urlRegex = /https?:\/\/[^\s)]+/g;
+            let match;
+            while ((match = urlRegex.exec(text)) !== null) {
+                let url = match[0];
+                // Remove trailing punctuation that shouldn't be part of the URL
+                url = url.replace(/[).,;!?]+$/, '');
+                urls.push({
+                    '@type': 'Entity',
+                    '@key': ['@type', 'url'],
+                    url,
+                });
+            }
+        }
+        return urls;
+    }
+    buildGroupDirectMessageActions(record) {
+        const actions = [];
+        const sender = this.buildPersonFromUserId(record.data.senderId);
+        // Build recipients from all participants except the sender
+        const participants = record.data.participants || [];
+        const recipients = participants
+            .filter((participantId) => participantId !== record.data.senderId)
+            .map((participantId) => this.buildPersonFromUserId(participantId));
+        const message = {
+            '@type': 'Message',
+            '@key': ['@type', 'source', 'sourceId'],
+            source: 'twitter',
+            sourceId: record.data.id,
+            body: record.data.text,
+            recipient: recipients,
+            author: [sender],
+        };
+        const sendAction = {
+            '@type': 'MessageAction',
+            '@key': ['@type', 'source', 'sourceId'],
+            source: 'twitter',
+            sourceId: record.data.id,
+            timestamp: record.data.created_at,
+            agent: sender,
+            object: message,
+        };
+        actions.push(sendAction);
+        return actions;
+    }
+}

package/dist/connectors/archive/TwitterArchiveDirectMessagesExtractor.d.ts

@@ -0,0 +1,38 @@
+import { Record } from '@chronicle.app/etl';
+import { z } from 'zod';
+import TwitterTransformer from '../TwitterTransformer.js';
+import { TwitterArchiveExtractor } from './TwitterArchiveExtractor.js';
+export declare class TwitterArchiveDirectMessagesExtractor extends TwitterArchiveExtractor<typeof TwitterArchiveDirectMessagesExtractor> {
+    static description: string;
+    static recordTypes: string[];
+    static defaultTransformer: typeof TwitterTransformer;
+    static newestFirst: boolean;
+    static schema: z.ZodObject<{
+        since: z.ZodOptional<z.ZodDate>;
+        until: z.ZodOptional<z.ZodDate>;
+        limit: z.ZodOptional<z.ZodNumber>;
+    } & {
+        input: z.ZodString;
+        'include-group-dms': z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
+    }, "strip", z.ZodTypeAny, {
+        input: string;
+        'include-group-dms': boolean;
+        since?: Date | undefined;
+        until?: Date | undefined;
+        limit?: number | undefined;
+    }, {
+        input: string;
+        'include-group-dms'?: boolean | undefined;
+        since?: Date | undefined;
+        until?: Date | undefined;
+        limit?: number | undefined;
+    }>;
+    /** The DM's message id — Twitter's own id, identical through any tool. */
+    keyOf(record: Record): string | null;
+    extract(): AsyncGenerator<Record>;
+    private loadDirectMessages;
+    private loadGroupHeaders;
+    private loadGroupDirectMessages;
+    determineCount(): Promise<number | null>;
+}
+//# sourceMappingURL=TwitterArchiveDirectMessagesExtractor.d.ts.map

package/dist/connectors/archive/TwitterArchiveDirectMessagesExtractor.js

@@ -0,0 +1,193 @@
+import { readFile } from 'node:fs/promises';
+import TwitterTransformer from '../TwitterTransformer.js';
+import { TwitterArchiveExtractor } from './TwitterArchiveExtractor.js';
+export class TwitterArchiveDirectMessagesExtractor extends TwitterArchiveExtractor {
+    static description = 'Direct messages';
+    static recordTypes = ['direct-messages', 'group-direct-messages'];
+    static defaultTransformer = TwitterTransformer;
+    // extract() sorts every message by created_at descending before yielding,
+    // which with keyOf opts this extractor into the frontier cursor.
+    static newestFirst = true;
+    static schema = TwitterArchiveExtractor.schema;
+    /** The DM's message id — Twitter's own id, identical through any tool. */
+    keyOf(record) {
+        return record.data.id ?? null;
+    }
+    async *extract() {
+        const config = this.config;
+        try {
+            // Load account info first
+            await this.loadAccountInfoFrom(config.input);
+            // Collect all messages from both sources
+            const allMessages = [];
+            // Process 1:1 direct messages
+            const oneOnOneMessages = await this.loadDirectMessages(`${config.input}/data/direct-messages.js`);
+            allMessages.push(...oneOnOneMessages);
+            // Process group direct messages (if enabled)
+            if (config['include-group-dms']) {
+                const groupMessages = await this.loadGroupDirectMessages(`${config.input}/data/direct-messages-group.js`);
+                allMessages.push(...groupMessages);
+            }
+            // Sort by timestamp descending (newest first)
+            allMessages.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
+            // Yield messages in chronological order (newest first)
+            let count = 0;
+            for (const message of allMessages) {
+                if (this.shouldStopExtracting(count))
+                    break;
+                if (this.isWithinDateRange(new Date(message.created_at))) {
+                    yield this.createRecordWithArchiveContext(message.data, message.metadata);
+                    count++;
+                }
+            }
+        }
+        catch (error) {
+            throw new Error(`Failed to process Twitter direct messages: ${error}`);
+        }
+    }
+    async loadDirectMessages(filePath) {
+        const messages = [];
+        try {
+            const content = await readFile(filePath, 'utf-8');
+            const jsData = this.parseTwitterJS(content, 'direct_messages');
+            for (const item of jsData) {
+                const conversation = item.dmConversation;
+                if (conversation && conversation.messages) {
+                    for (const message of conversation.messages) {
+                        const msg = message.messageCreate;
+                        if (msg) {
+                            messages.push({
+                                created_at: msg.createdAt,
+                                data: {
+                                    type: 'direct-message',
+                                    id: msg.id,
+                                    conversationId: conversation.conversationId,
+                                    senderId: msg.senderId,
+                                    recipientId: msg.recipientId,
+                                    text: msg.text,
+                                    created_at: msg.createdAt,
+                                    mediaUrls: msg.mediaUrls,
+                                    urls: msg.urls,
+                                },
+                                metadata: { originalData: msg },
+                            });
+                        }
+                    }
+                }
+            }
+        }
+        catch (error) {
+            console.warn(`Could not process 1:1 direct messages: ${error}`);
+        }
+        return messages;
+    }
+    async loadGroupHeaders(filePath) {
+        const groupHeaders = new Map();
+        try {
+            const content = await readFile(filePath, 'utf-8');
+            const jsData = this.parseTwitterJS(content, 'direct_message_group_headers');
+            for (const item of jsData) {
+                const conversation = item.dmConversation;
+                if (conversation) {
+                    // Find the joinConversation event to get participants
+                    const joinEvent = conversation.messages?.find((msg) => msg.joinConversation);
+                    if (joinEvent) {
+                        groupHeaders.set(conversation.conversationId, {
+                            conversationId: conversation.conversationId,
+                            participants: joinEvent.joinConversation.participantsSnapshot || [],
+                            initiator: joinEvent.joinConversation.initiatingUserId,
+                        });
+                    }
+                }
+            }
+        }
+        catch (error) {
+            console.warn(`Could not load group headers: ${error}`);
+        }
+        return groupHeaders;
+    }
+    async loadGroupDirectMessages(filePath) {
+        const messages = [];
+        try {
+            // First load group headers to get participant info
+            const groupHeaders = await this.loadGroupHeaders(`${filePath.replace('direct-messages-group.js', 'direct-message-group-headers.js')}`);
+            const content = await readFile(filePath, 'utf-8');
+            const jsData = this.parseTwitterJS(content, 'direct_messages_group');
+            for (const item of jsData) {
+                const conversation = item.dmConversation;
+                if (conversation && conversation.messages) {
+                    const groupInfo = groupHeaders.get(conversation.conversationId);
+                    for (const message of conversation.messages) {
+                        // Only process messageCreate events (actual messages), skip other events
+                        if (message.messageCreate) {
+                            const msg = message.messageCreate;
+                            messages.push({
+                                created_at: msg.createdAt,
+                                data: {
+                                    type: 'group-direct-message',
+                                    id: msg.id,
+                                    conversationId: conversation.conversationId,
+                                    senderId: msg.senderId,
+                                    participants: groupInfo?.participants || [],
+                                    text: msg.text || '',
+                                    created_at: msg.createdAt,
+                                    mediaUrls: msg.mediaUrls || [],
+                                    urls: msg.urls || [],
+                                },
+                                metadata: {
+                                    recordType: 'group-direct-messages',
+                                    originalData: msg,
+                                    groupInfo,
+                                },
+                            });
+                        }
+                    }
+                }
+            }
+        }
+        catch (error) {
+            console.warn(`Could not process group direct messages: ${error}`);
+        }
+        return messages;
+    }
+    async determineCount() {
+        const config = this.config;
+        try {
+            let totalMessages = 0;
+            // Count 1:1 direct messages
+            try {
+                const dmContent = await readFile(`${config.input}/data/direct-messages.js`, 'utf-8');
+                const dmData = this.parseTwitterJS(dmContent, 'direct_messages');
+                for (const item of dmData) {
+                    const conversation = item.dmConversation;
+                    if (conversation && conversation.messages) {
+                        totalMessages += conversation.messages.filter((msg) => msg.messageCreate).length;
+                    }
+                }
+            }
+            catch {
+                // 1:1 DMs file may not exist
+            }
+            // Count group direct messages (if enabled)
+            if (config['include-group-dms']) {
+                try {
+                    const groupContent = await readFile(`${config.input}/data/direct-messages-group.js`, 'utf-8');
+                    const groupData = this.parseTwitterJS(groupContent, 'direct_messages_group');
+                    for (const item of groupData) {
+                        const conversation = item.dmConversation;
+                        if (conversation && conversation.messages) {
+                            totalMessages += conversation.messages.filter((msg) => msg.messageCreate).length;
+                        }
+                    }
+                }
+                catch {
+                    // Group DMs file may not exist
+                }
+            }
+            return totalMessages;
+        }
+        catch {
+            return null;
+        }
+    }
+}

package/dist/connectors/archive/TwitterArchiveExtractor.d.ts

@@ -0,0 +1,36 @@
+import { ArchiveExtractor } from '@chronicle.app/etl';
+import { z } from 'zod';
+export declare abstract class TwitterArchiveExtractor<T extends typeof TwitterArchiveExtractor> extends ArchiveExtractor<T> {
+    static source: string;
+    static strategy: string;
+    static schema: z.ZodObject<{
+        since: z.ZodOptional<z.ZodDate>;
+        until: z.ZodOptional<z.ZodDate>;
+        limit: z.ZodOptional<z.ZodNumber>;
+    } & {
+        input: z.ZodString;
+        'include-group-dms': z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
+    }, "strip", z.ZodTypeAny, {
+        input: string;
+        'include-group-dms': boolean;
+        since?: Date | undefined;
+        until?: Date | undefined;
+        limit?: number | undefined;
+    }, {
+        input: string;
+        'include-group-dms'?: boolean | undefined;
+        since?: Date | undefined;
+        until?: Date | undefined;
+        limit?: number | undefined;
+    }>;
+    /**
+     * Twitter's export needs the archive's base directory to locate
+     * `data/account.js`, so this can't share the base class's no-arg
+     * `loadAccountInfo()` signature — kept as its own method instead of an
+     * incompatible override.
+     */
+    protected loadAccountInfoFrom(basePath: string): Promise<void>;
+    protected parseTwitterJS(content: string, dataType: string): any[];
+    protected convertTwitterDateToISO(twitterDate: string): string;
+}
+//# sourceMappingURL=TwitterArchiveExtractor.d.ts.map

package/dist/connectors/archive/TwitterArchiveExtractor.js

@@ -0,0 +1,51 @@
+import { ArchiveExtractor, Extractor } from '@chronicle.app/etl';
+import { readFile } from 'node:fs/promises';
+import { z } from 'zod';
+export class TwitterArchiveExtractor extends ArchiveExtractor {
+    static source = 'twitter';
+    static strategy = 'archive';
+    static schema = Extractor.schema.extend({
+        input: z.string().describe('Path to Twitter export data directory'),
+        'include-group-dms': z
+            .boolean()
+            .optional()
+            .default(true)
+            .describe('Include group direct messages in extraction'),
+    });
+    /**
+     * Twitter's export needs the archive's base directory to locate
+     * `data/account.js`, so this can't share the base class's no-arg
+     * `loadAccountInfo()` signature — kept as its own method instead of an
+     * incompatible override.
+     */
+    async loadAccountInfoFrom(basePath) {
+        try {
+            const content = await readFile(`${basePath}/data/account.js`, 'utf-8');
+            const jsData = this.parseTwitterJS(content, 'account');
+            this.accountInfo = jsData[0]?.account || null;
+        }
+        catch (error) {
+            console.warn(`Could not load account info: ${error}`);
+            this.accountInfo = null;
+        }
+    }
+    parseTwitterJS(content, dataType) {
+        const regex = new RegExp(`window\\.YTD\\.${dataType}\\.part0\\s*=\\s*(.+)`, 's');
+        const match = content.match(regex);
+        if (!match) {
+            throw new Error(`Could not parse ${dataType} data from Twitter export`);
+        }
+        try {
+            return JSON.parse(match[1]);
+        }
+        catch (error) {
+            throw new Error(`Invalid JSON in ${dataType} file: ${error}`);
+        }
+    }
+    convertTwitterDateToISO(twitterDate) {
+        // Convert Twitter's date format "Sat Jan 11 17:10:28 +0000 2025" to ISO
+        // Twitter uses format: "ddd MMM DD HH:mm:ss +0000 YYYY"
+        const date = new Date(twitterDate);
+        return date.toISOString();
+    }
+}

package/dist/connectors/archive/TwitterArchiveLikesExtractor.d.ts

@@ -0,0 +1,35 @@
+import { Record } from '@chronicle.app/etl';
+import { z } from 'zod';
+import TwitterTransformer from '../TwitterTransformer.js';
+import { TwitterArchiveExtractor } from './TwitterArchiveExtractor.js';
+export declare class TwitterArchiveLikesExtractor extends TwitterArchiveExtractor<typeof TwitterArchiveLikesExtractor> {
+    static description: string;
+    static recordTypes: string[];
+    static defaultTransformer: typeof TwitterTransformer;
+    static schema: z.ZodObject<{
+        since: z.ZodOptional<z.ZodDate>;
+        until: z.ZodOptional<z.ZodDate>;
+        limit: z.ZodOptional<z.ZodNumber>;
+    } & {
+        input: z.ZodString;
+        'include-group-dms': z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
+    }, "strip", z.ZodTypeAny, {
+        input: string;
+        'include-group-dms': boolean;
+        since?: Date | undefined;
+        until?: Date | undefined;
+        limit?: number | undefined;
+    }, {
+        input: string;
+        'include-group-dms'?: boolean | undefined;
+        since?: Date | undefined;
+        until?: Date | undefined;
+        limit?: number | undefined;
+    }>;
+    /** The liked tweet's id — the like entry's only id, and Twitter's own. */
+    keyOf(record: Record): string | null;
+    extract(): AsyncGenerator<Record>;
+    private processLikes;
+    determineCount(): Promise<number | null>;
+}
+//# sourceMappingURL=TwitterArchiveLikesExtractor.d.ts.map

package/dist/connectors/archive/TwitterArchiveLikesExtractor.js

@@ -0,0 +1,59 @@
+import { readFile } from 'node:fs/promises';
+import TwitterTransformer from '../TwitterTransformer.js';
+import { TwitterArchiveExtractor } from './TwitterArchiveExtractor.js';
+export class TwitterArchiveLikesExtractor extends TwitterArchiveExtractor {
+    static description = 'Likes';
+    static recordTypes = ['likes'];
+    static defaultTransformer = TwitterTransformer;
+    static schema = TwitterArchiveExtractor.schema;
+    /** The liked tweet's id — the like entry's only id, and Twitter's own. */
+    keyOf(record) {
+        return record.data.tweetId ?? null;
+    }
+    async *extract() {
+        const config = this.config;
+        try {
+            // Load account info first
+            await this.loadAccountInfoFrom(config.input);
+            yield* this.processLikes(`${config.input}/data/like.js`);
+        }
+        catch (error) {
+            throw new Error(`Failed to process Twitter likes: ${error}`);
+        }
+    }
+    async *processLikes(filePath) {
+        try {
+            const content = await readFile(filePath, 'utf-8');
+            const jsData = this.parseTwitterJS(content, 'like');
+            let count = 0;
+            for (const item of jsData) {
+                const { like } = item;
+                if (like) {
+                    if (this.shouldStopExtracting(count))
+                        break;
+                    yield this.createRecordWithArchiveContext({
+                        type: 'like',
+                        tweetId: like.tweetId,
+                        fullText: like.fullText,
+                        expandedUrl: like.expandedUrl,
+                    }, { originalData: like });
+                    count++;
+                }
+            }
+        }
+        catch (error) {
+            console.warn(`Could not process likes: ${error}`);
+        }
+    }
+    async determineCount() {
+        const config = this.config;
+        try {
+            const likesContent = await readFile(`${config.input}/data/like.js`, 'utf-8');
+            const likesData = this.parseTwitterJS(likesContent, 'like');
+            return likesData.length;
+        }
+        catch {
+            return null;
+        }
+    }
+}

package/dist/connectors/archive/TwitterArchiveTweetsExtractor.d.ts

@@ -0,0 +1,36 @@
+import { Record } from '@chronicle.app/etl';
+import { z } from 'zod';
+import TwitterTransformer from '../TwitterTransformer.js';
+import { TwitterArchiveExtractor } from './TwitterArchiveExtractor.js';
+export declare class TwitterArchiveTweetsExtractor extends TwitterArchiveExtractor<typeof TwitterArchiveTweetsExtractor> {
+    static description: string;
+    static recordTypes: string[];
+    static default: boolean;
+    static defaultTransformer: typeof TwitterTransformer;
+    static schema: z.ZodObject<{
+        since: z.ZodOptional<z.ZodDate>;
+        until: z.ZodOptional<z.ZodDate>;
+        limit: z.ZodOptional<z.ZodNumber>;
+    } & {
+        input: z.ZodString;
+        'include-group-dms': z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
+    }, "strip", z.ZodTypeAny, {
+        input: string;
+        'include-group-dms': boolean;
+        since?: Date | undefined;
+        until?: Date | undefined;
+        limit?: number | undefined;
+    }, {
+        input: string;
+        'include-group-dms'?: boolean | undefined;
+        since?: Date | undefined;
+        until?: Date | undefined;
+        limit?: number | undefined;
+    }>;
+    /** The tweet id (`id_str`) — Twitter's own id, identical through any tool. */
+    keyOf(record: Record): string | null;
+    extract(): AsyncGenerator<Record>;
+    private processTweets;
+    determineCount(): Promise<number | null>;
+}
+//# sourceMappingURL=TwitterArchiveTweetsExtractor.d.ts.map

package/dist/connectors/archive/TwitterArchiveTweetsExtractor.js

@@ -0,0 +1,67 @@
+import { readFile } from 'node:fs/promises';
+import TwitterTransformer from '../TwitterTransformer.js';
+import { TwitterArchiveExtractor } from './TwitterArchiveExtractor.js';
+export class TwitterArchiveTweetsExtractor extends TwitterArchiveExtractor {
+    static description = 'Tweets';
+    static recordTypes = ['tweets'];
+    static default = true;
+    static defaultTransformer = TwitterTransformer;
+    static schema = TwitterArchiveExtractor.schema;
+    /** The tweet id (`id_str`) — Twitter's own id, identical through any tool. */
+    keyOf(record) {
+        return record.data.id ?? null;
+    }
+    async *extract() {
+        const config = this.config;
+        try {
+            // Load account info first
+            await this.loadAccountInfoFrom(config.input);
+            yield* this.processTweets(`${config.input}/data/tweets.js`);
+        }
+        catch (error) {
+            throw new Error(`Failed to process Twitter tweets: ${error}`);
+        }
+    }
+    async *processTweets(filePath) {
+        try {
+            const content = await readFile(filePath, 'utf-8');
+            const jsData = this.parseTwitterJS(content, 'tweets');
+            let count = 0;
+            for (const item of jsData) {
+                const { tweet } = item;
+                if (tweet && this.isWithinDateRange(new Date(tweet.created_at))) {
+                    if (this.shouldStopExtracting(count))
+                        break;
+                    yield this.createRecordWithArchiveContext({
+                        type: 'tweet',
+                        id: tweet.id_str,
+                        text: tweet.full_text || tweet.text,
+                        created_at: this.convertTwitterDateToISO(tweet.created_at),
+                        favorite_count: Number.parseInt(tweet.favorite_count) || 0,
+                        retweet_count: Number.parseInt(tweet.retweet_count) || 0,
+                        in_reply_to_status_id: tweet.in_reply_to_status_id_str,
+                        in_reply_to_user_id: tweet.in_reply_to_user_id_str,
+                        entities: tweet.entities,
+                        source: tweet.source,
+                        retweeted: tweet.retweeted,
+                    }, { originalData: tweet });
+                    count++;
+                }
+            }
+        }
+        catch (error) {
+            console.warn(`Could not process tweets: ${error}`);
+        }
+    }
+    async determineCount() {
+        const config = this.config;
+        try {
+            const tweetsContent = await readFile(`${config.input}/data/tweets.js`, 'utf-8');
+            const tweetsData = this.parseTwitterJS(tweetsContent, 'tweets');
+            return tweetsData.length;
+        }
+        catch {
+            return null;
+        }
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,6 @@
+export { TwitterArchiveExtractor } from './connectors/archive/TwitterArchiveExtractor.js';
+export { TwitterArchiveTweetsExtractor } from './connectors/archive/TwitterArchiveTweetsExtractor.js';
+export { TwitterArchiveLikesExtractor } from './connectors/archive/TwitterArchiveLikesExtractor.js';
+export { TwitterArchiveDirectMessagesExtractor } from './connectors/archive/TwitterArchiveDirectMessagesExtractor.js';
+export { default as TwitterTransformer } from './connectors/TwitterTransformer.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.js

@@ -0,0 +1,5 @@
+export { TwitterArchiveExtractor } from './connectors/archive/TwitterArchiveExtractor.js';
+export { TwitterArchiveTweetsExtractor } from './connectors/archive/TwitterArchiveTweetsExtractor.js';
+export { TwitterArchiveLikesExtractor } from './connectors/archive/TwitterArchiveLikesExtractor.js';
+export { TwitterArchiveDirectMessagesExtractor } from './connectors/archive/TwitterArchiveDirectMessagesExtractor.js';
+export { default as TwitterTransformer } from './connectors/TwitterTransformer.js';

package/package.json

@@ -0,0 +1,76 @@
+{
+  "name": "@chronicle.app/twitter",
+  "version": "0.3.0",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist/**/*.js",
+    "dist/**/*.d.ts"
+  ],
+  "scripts": {
+    "build": "tsc -b tsconfig.lib.json",
+    "typecheck": "tsc -p tsconfig.json",
+    "lint": "eslint src --ext .ts,.js",
+    "test": "node --test src/*.test.js"
+  },
+  "keywords": [
+    "chronicle",
+    "twitter"
+  ],
+  "author": "Andrew Louis",
+  "license": "MIT",
+  "type": "module",
+  "description": "Chronicle plugin for twitter",
+  "devDependencies": {
+    "@chronicle.app/etl": "0.3.0",
+    "@chronicle.app/schema": "0.3.0",
+    "@chronicle.app/tsconfig": "0.3.0",
+    "@types/node": "^22.10.1",
+    "typescript": "^5.7.2"
+  },
+  "homepage": "https://chronicle.app",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/chronicle-app/chronicle.git",
+    "directory": "plugins/twitter"
+  },
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
+  },
+  "dependencies": {
+    "zod": "^3.24.1"
+  },
+  "peerDependencies": {
+    "@chronicle.app/etl": ">=0.1.0 <1.0.0",
+    "@chronicle.app/schema": ">=0.1.0 <1.0.0"
+  },
+  "engines": {
+    "node": ">=22.13.0"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "chronicle": {
+    "plugin": true,
+    "sources": {
+      "twitter": {
+        "strategies": {
+          "archive": {
+            "delivery": "export",
+            "recordTypes": [
+              "direct-messages",
+              "group-direct-messages",
+              "likes",
+              "tweets"
+            ],
+            "default": true
+          }
+        }
+      }
+    }
+  }
+}