@chronicle.app/twitter 0.3.0 → 0.5.0

package/README.md

@@ -30,7 +30,7 @@ chronicle extract twitter \
 
 The extractor supports the following options:
 
-- `--input` (required): Path to the Twitter export data directory
+- `--input` (required): The Twitter/X archive, as a zip or its unzipped folder
 - `--type`: Which record types to read: `tweets`, `likes`, `direct-messages`,
   `group-direct-messages` (default: all of them)
 - `--include-group-dms`: Include group direct messages (default: true)
@@ -50,7 +50,7 @@ npm test
 
 ## File Format
 
-The input is the unzipped directory of a Twitter/X data archive. The plugin
+The input is a Twitter/X data archive, zipped or unzipped. The plugin
 reads these files from its `data/` folder, each a `window.YTD.<name>.part0 = [...]`
 script wrapping a JSON array:
 
@@ -63,10 +63,10 @@ script wrapping a JSON array:
 
 ## Schema
 
-- Each tweet is a `PublishAction` whose object is a `Post`. The post's
+- Each tweet is a `PublishAction` whose `result` is a `Post`, keyed by it. The post's
   `references` are the accounts it mentions and the links it contains.
 - Each like is a `LikeAction` on the liked `Post`.
-- Each direct message is a `MessageAction` whose object is a `Message`, sent to
+- Each direct message is a `MessageAction` whose `result` is a `Message`, keyed by it, sent to
   the other participants of the conversation.
 
 The archive owner is an `Agent` linked to `@me`; other accounts are `Agent`s

package/dist/connectors/TwitterTransformer.js

@@ -35,12 +35,12 @@ export default class TwitterTransformer extends ChronicleTransformer {
         };
         const publishAction = {
             '@type': 'PublishAction',
-            '@key': ['@type', 'source', 'sourceId'],
+            // Keyed by the tweet it published.
+            '@key': ['@type', 'source', 'result.sourceId'],
             source: 'twitter',
-            sourceId: record.data.id,
             timestamp: record.data.created_at,
             agent: user,
-            object: tweet,
+            result: tweet,
         };
         actions.push(publishAction);
         return actions;
@@ -57,9 +57,9 @@ export default class TwitterTransformer extends ChronicleTransformer {
         };
         const likeAction = {
             '@type': 'LikeAction',
-            '@key': ['@type', 'source', 'sourceId'],
+            // You like a tweet once, so the tweet identifies the like.
+            '@key': ['@type', 'source', 'object.sourceId'],
             source: 'twitter',
-            sourceId: `${record.data.tweetId}-like`,
             agent: user,
             object: targetTweet,
         };
@@ -82,12 +82,12 @@ export default class TwitterTransformer extends ChronicleTransformer {
         // Media URLs would need to be handled separately or added to schema
         const sendAction = {
             '@type': 'MessageAction',
-            '@key': ['@type', 'source', 'sourceId'],
+            // Keyed by the message it sent.
+            '@key': ['@type', 'source', 'result.sourceId'],
             source: 'twitter',
-            sourceId: record.data.id,
             timestamp: record.data.created_at,
             agent: sender,
-            object: message,
+            result: message,
         };
         actions.push(sendAction);
         return actions;
@@ -200,12 +200,12 @@ export default class TwitterTransformer extends ChronicleTransformer {
         };
         const sendAction = {
             '@type': 'MessageAction',
-            '@key': ['@type', 'source', 'sourceId'],
+            // Keyed by the message it sent.
+            '@key': ['@type', 'source', 'result.sourceId'],
             source: 'twitter',
-            sourceId: record.data.id,
             timestamp: record.data.created_at,
             agent: sender,
-            object: message,
+            result: message,
         };
         actions.push(sendAction);
         return actions;

package/dist/connectors/archive/TwitterArchiveDirectMessagesExtractor.d.ts

@@ -1,4 +1,4 @@
-import { Record } from '@chronicle.app/etl';
+import { Record, type Survey, type SurveyOptions } from '@chronicle.app/etl';
 import { z } from 'zod';
 import TwitterTransformer from '../TwitterTransformer.js';
 import { TwitterArchiveExtractor } from './TwitterArchiveExtractor.js';
@@ -30,9 +30,13 @@ export declare class TwitterArchiveDirectMessagesExtractor extends TwitterArchiv
     /** The DM's message id — Twitter's own id, identical through any tool. */
     keyOf(record: Record): string | null;
     extract(): AsyncGenerator<Record>;
+    /** The 1:1 messages, and group messages if enabled, within since and until. */
+    private loadMessages;
     private loadDirectMessages;
     private loadGroupHeaders;
     private loadGroupDirectMessages;
     determineCount(): Promise<number | null>;
+    /** The messages a run reads, when they were sent, and in how many conversations. */
+    survey(options?: SurveyOptions): Promise<Survey>;
 }
 //# sourceMappingURL=TwitterArchiveDirectMessagesExtractor.d.ts.map

package/dist/connectors/archive/TwitterArchiveDirectMessagesExtractor.js

@@ -1,4 +1,3 @@
-import { readFile } from 'node:fs/promises';
 import TwitterTransformer from '../TwitterTransformer.js';
 import { TwitterArchiveExtractor } from './TwitterArchiveExtractor.js';
 export class TwitterArchiveDirectMessagesExtractor extends TwitterArchiveExtractor {
@@ -14,20 +13,11 @@ export class TwitterArchiveDirectMessagesExtractor extends TwitterArchiveExtract
         return record.data.id ?? null;
     }
     async *extract() {
-        const config = this.config;
         try {
             // Load account info first
-            await this.loadAccountInfoFrom(config.input);
+            this.accountInfo = await this.loadAccountInfo();
             // Collect all messages from both sources
-            const allMessages = [];
-            // Process 1:1 direct messages
-            const oneOnOneMessages = await this.loadDirectMessages(`${config.input}/data/direct-messages.js`);
-            allMessages.push(...oneOnOneMessages);
-            // Process group direct messages (if enabled)
-            if (config['include-group-dms']) {
-                const groupMessages = await this.loadGroupDirectMessages(`${config.input}/data/direct-messages-group.js`);
-                allMessages.push(...groupMessages);
-            }
+            const allMessages = await this.loadMessages();
             // Sort by timestamp descending (newest first)
             allMessages.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
             // Yield messages in chronological order (newest first)
@@ -35,20 +25,29 @@ export class TwitterArchiveDirectMessagesExtractor extends TwitterArchiveExtract
             for (const message of allMessages) {
                 if (this.shouldStopExtracting(count))
                     break;
-                if (this.isWithinDateRange(new Date(message.created_at))) {
-                    yield this.createRecordWithArchiveContext(message.data, message.metadata);
-                    count++;
-                }
+                yield this.createRecordWithArchiveContext(message.data, message.metadata);
+                count++;
             }
         }
         catch (error) {
             throw new Error(`Failed to process Twitter direct messages: ${error}`);
         }
     }
+    /** The 1:1 messages, and group messages if enabled, within since and until. */
+    async loadMessages() {
+        const config = this.config;
+        // Process 1:1 direct messages
+        const messages = await this.loadDirectMessages(`data/direct-messages.js`);
+        // Process group direct messages (if enabled)
+        if (config['include-group-dms']) {
+            messages.push(...(await this.loadGroupDirectMessages(`data/direct-messages-group.js`)));
+        }
+        return messages.filter(message => this.isWithinDateRange(new Date(message.created_at)));
+    }
     async loadDirectMessages(filePath) {
         const messages = [];
         try {
-            const content = await readFile(filePath, 'utf-8');
+            const content = await this.files.readText(filePath);
             const jsData = this.parseTwitterJS(content, 'direct_messages');
             for (const item of jsData) {
                 const conversation = item.dmConversation;
@@ -77,14 +76,14 @@ export class TwitterArchiveDirectMessagesExtractor extends TwitterArchiveExtract
             }
         }
         catch (error) {
-            console.warn(`Could not process 1:1 direct messages: ${error}`);
+            this.logger.warn('Could not process 1:1 direct messages', { error: String(error) });
         }
         return messages;
     }
     async loadGroupHeaders(filePath) {
         const groupHeaders = new Map();
         try {
-            const content = await readFile(filePath, 'utf-8');
+            const content = await this.files.readText(filePath);
             const jsData = this.parseTwitterJS(content, 'direct_message_group_headers');
             for (const item of jsData) {
                 const conversation = item.dmConversation;
@@ -102,7 +101,7 @@ export class TwitterArchiveDirectMessagesExtractor extends TwitterArchiveExtract
             }
         }
         catch (error) {
-            console.warn(`Could not load group headers: ${error}`);
+            this.logger.warn('Could not load group headers', { error: String(error) });
         }
         return groupHeaders;
     }
@@ -111,7 +110,7 @@ export class TwitterArchiveDirectMessagesExtractor extends TwitterArchiveExtract
         try {
             // First load group headers to get participant info
             const groupHeaders = await this.loadGroupHeaders(`${filePath.replace('direct-messages-group.js', 'direct-message-group-headers.js')}`);
-            const content = await readFile(filePath, 'utf-8');
+            const content = await this.files.readText(filePath);
             const jsData = this.parseTwitterJS(content, 'direct_messages_group');
             for (const item of jsData) {
                 const conversation = item.dmConversation;
@@ -146,7 +145,7 @@ export class TwitterArchiveDirectMessagesExtractor extends TwitterArchiveExtract
             }
         }
         catch (error) {
-            console.warn(`Could not process group direct messages: ${error}`);
+            this.logger.warn('Could not process group direct messages', { error: String(error) });
         }
         return messages;
     }
@@ -156,7 +155,7 @@ export class TwitterArchiveDirectMessagesExtractor extends TwitterArchiveExtract
             let totalMessages = 0;
             // Count 1:1 direct messages
             try {
-                const dmContent = await readFile(`${config.input}/data/direct-messages.js`, 'utf-8');
+                const dmContent = await this.files.readText(`data/direct-messages.js`);
                 const dmData = this.parseTwitterJS(dmContent, 'direct_messages');
                 for (const item of dmData) {
                     const conversation = item.dmConversation;
@@ -171,7 +170,7 @@ export class TwitterArchiveDirectMessagesExtractor extends TwitterArchiveExtract
             // Count group direct messages (if enabled)
             if (config['include-group-dms']) {
                 try {
-                    const groupContent = await readFile(`${config.input}/data/direct-messages-group.js`, 'utf-8');
+                    const groupContent = await this.files.readText(`data/direct-messages-group.js`);
                     const groupData = this.parseTwitterJS(groupContent, 'direct_messages_group');
                     for (const item of groupData) {
                         const conversation = item.dmConversation;
@@ -190,4 +189,23 @@ export class TwitterArchiveDirectMessagesExtractor extends TwitterArchiveExtract
             return null;
         }
     }
+    /** The messages a run reads, when they were sent, and in how many conversations. */
+    async survey(options = {}) {
+        const config = this.config;
+        const messages = await this.loadMessages();
+        const survey = {};
+        for (const recordType of TwitterArchiveDirectMessagesExtractor.recordTypes) {
+            if (recordType === 'group-direct-messages' && !config['include-group-dms'])
+                continue;
+            // A 1:1 message names no record type, so its record is the first, as in a run.
+            const sent = messages.filter(message => (message.metadata.recordType ?? 'direct-messages') === recordType);
+            const times = sent.map(message => new Date(message.created_at));
+            const conversations = new Set(sent.map(message => message.data.conversationId)).size;
+            survey[recordType] = {
+                ...this.surveyOf(times, options),
+                measures: [{ key: 'conversations', label: 'conversations', value: conversations }],
+            };
+        }
+        return survey;
+    }
 }

package/dist/connectors/archive/TwitterArchiveExtractor.d.ts

@@ -1,4 +1,4 @@
-import { ArchiveExtractor } from '@chronicle.app/etl';
+import { ArchiveExtractor, type KindSurvey, type SurveyOptions } from '@chronicle.app/etl';
 import { z } from 'zod';
 export declare abstract class TwitterArchiveExtractor<T extends typeof TwitterArchiveExtractor> extends ArchiveExtractor<T> {
     static source: string;
@@ -23,14 +23,12 @@ export declare abstract class TwitterArchiveExtractor<T extends typeof TwitterAr
         until?: Date | undefined;
         limit?: number | undefined;
     }>;
-    /**
-     * Twitter's export needs the archive's base directory to locate
-     * `data/account.js`, so this can't share the base class's no-arg
-     * `loadAccountInfo()` signature — kept as its own method instead of an
-     * incompatible override.
-     */
-    protected loadAccountInfoFrom(basePath: string): Promise<void>;
+    protected readonly exportName = "Twitter archive";
+    /** Whose archive this is, from `data/account.js`. */
+    protected loadAccountInfo(): Promise<any>;
     protected parseTwitterJS(content: string, dataType: string): any[];
+    /** How many records a run reads and when, from the time it dates each by. */
+    protected surveyOf(times: Date[], options: SurveyOptions): KindSurvey;
     protected convertTwitterDateToISO(twitterDate: string): string;
 }
 //# sourceMappingURL=TwitterArchiveExtractor.d.ts.map

package/dist/connectors/archive/TwitterArchiveExtractor.js

@@ -1,32 +1,27 @@
-import { ArchiveExtractor, Extractor } from '@chronicle.app/etl';
-import { readFile } from 'node:fs/promises';
+import { ArchiveExtractor, countMonth, Extractor, } from '@chronicle.app/etl';
 import { z } from 'zod';
 export class TwitterArchiveExtractor extends ArchiveExtractor {
     static source = 'twitter';
     static strategy = 'archive';
     static schema = Extractor.schema.extend({
-        input: z.string().describe('Path to Twitter export data directory'),
+        input: z.string().describe('The Twitter archive, as a zip or its unpacked folder'),
         'include-group-dms': z
             .boolean()
             .optional()
             .default(true)
             .describe('Include group direct messages in extraction'),
     });
-    /**
-     * Twitter's export needs the archive's base directory to locate
-     * `data/account.js`, so this can't share the base class's no-arg
-     * `loadAccountInfo()` signature — kept as its own method instead of an
-     * incompatible override.
-     */
-    async loadAccountInfoFrom(basePath) {
+    exportName = 'Twitter archive';
+    /** Whose archive this is, from `data/account.js`. */
+    async loadAccountInfo() {
         try {
-            const content = await readFile(`${basePath}/data/account.js`, 'utf-8');
+            const content = await this.files.readText('data/account.js');
             const jsData = this.parseTwitterJS(content, 'account');
-            this.accountInfo = jsData[0]?.account || null;
+            return jsData[0]?.account || null;
         }
         catch (error) {
-            console.warn(`Could not load account info: ${error}`);
-            this.accountInfo = null;
+            this.logger.warn('Could not load account info', { error: String(error) });
+            return null;
         }
     }
     parseTwitterJS(content, dataType) {
@@ -42,6 +37,19 @@ export class TwitterArchiveExtractor extends ArchiveExtractor {
             throw new Error(`Invalid JSON in ${dataType} file: ${error}`);
         }
     }
+    /** How many records a run reads and when, from the time it dates each by. */
+    surveyOf(times, options) {
+        const kind = { count: times.length, ...(options.byMonth && { byMonth: {} }) };
+        for (const time of times) {
+            if (!kind.earliest || time < kind.earliest)
+                kind.earliest = time;
+            if (!kind.latest || time > kind.latest)
+                kind.latest = time;
+            if (kind.byMonth)
+                countMonth(kind.byMonth, time);
+        }
+        return kind;
+    }
     convertTwitterDateToISO(twitterDate) {
         // Convert Twitter's date format "Sat Jan 11 17:10:28 +0000 2025" to ISO
         // Twitter uses format: "ddd MMM DD HH:mm:ss +0000 YYYY"

package/dist/connectors/archive/TwitterArchiveLikesExtractor.d.ts

@@ -1,19 +1,18 @@
-import { Record } from '@chronicle.app/etl';
-import { z } from 'zod';
+import { Record, type Survey } from '@chronicle.app/etl';
 import TwitterTransformer from '../TwitterTransformer.js';
 import { TwitterArchiveExtractor } from './TwitterArchiveExtractor.js';
 export declare class TwitterArchiveLikesExtractor extends TwitterArchiveExtractor<typeof TwitterArchiveLikesExtractor> {
     static description: string;
     static recordTypes: string[];
     static defaultTransformer: typeof TwitterTransformer;
-    static schema: z.ZodObject<{
-        since: z.ZodOptional<z.ZodDate>;
-        until: z.ZodOptional<z.ZodDate>;
-        limit: z.ZodOptional<z.ZodNumber>;
+    static schema: import("zod").ZodObject<{
+        since: import("zod").ZodOptional<import("zod").ZodDate>;
+        until: import("zod").ZodOptional<import("zod").ZodDate>;
+        limit: import("zod").ZodOptional<import("zod").ZodNumber>;
     } & {
-        input: z.ZodString;
-        'include-group-dms': z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
-    }, "strip", z.ZodTypeAny, {
+        input: import("zod").ZodString;
+        'include-group-dms': import("zod").ZodDefault<import("zod").ZodOptional<import("zod").ZodBoolean>>;
+    }, "strip", import("zod").ZodTypeAny, {
         input: string;
         'include-group-dms': boolean;
         since?: Date | undefined;
@@ -30,6 +29,10 @@ export declare class TwitterArchiveLikesExtractor extends TwitterArchiveExtracto
     keyOf(record: Record): string | null;
     extract(): AsyncGenerator<Record>;
     private processLikes;
+    /** The archive's likes, as it writes them: with no time, so since and until can't apply. */
+    private loadLikes;
     determineCount(): Promise<number | null>;
+    /** The likes a run reads, counted alone: the archive doesn't say when any were made. */
+    survey(): Promise<Survey>;
 }
 //# sourceMappingURL=TwitterArchiveLikesExtractor.d.ts.map

package/dist/connectors/archive/TwitterArchiveLikesExtractor.js

@@ -1,4 +1,3 @@
-import { readFile } from 'node:fs/promises';
 import TwitterTransformer from '../TwitterTransformer.js';
 import { TwitterArchiveExtractor } from './TwitterArchiveExtractor.js';
 export class TwitterArchiveLikesExtractor extends TwitterArchiveExtractor {
@@ -11,11 +10,10 @@ export class TwitterArchiveLikesExtractor extends TwitterArchiveExtractor {
         return record.data.tweetId ?? null;
     }
     async *extract() {
-        const config = this.config;
         try {
             // Load account info first
-            await this.loadAccountInfoFrom(config.input);
-            yield* this.processLikes(`${config.input}/data/like.js`);
+            this.accountInfo = await this.loadAccountInfo();
+            yield* this.processLikes(`data/like.js`);
         }
         catch (error) {
             throw new Error(`Failed to process Twitter likes: ${error}`);
@@ -23,32 +21,33 @@ export class TwitterArchiveLikesExtractor extends TwitterArchiveExtractor {
     }
     async *processLikes(filePath) {
         try {
-            const content = await readFile(filePath, 'utf-8');
-            const jsData = this.parseTwitterJS(content, 'like');
             let count = 0;
-            for (const item of jsData) {
-                const { like } = item;
-                if (like) {
-                    if (this.shouldStopExtracting(count))
-                        break;
-                    yield this.createRecordWithArchiveContext({
-                        type: 'like',
-                        tweetId: like.tweetId,
-                        fullText: like.fullText,
-                        expandedUrl: like.expandedUrl,
-                    }, { originalData: like });
-                    count++;
-                }
+            for (const like of await this.loadLikes(filePath)) {
+                if (this.shouldStopExtracting(count))
+                    break;
+                yield this.createRecordWithArchiveContext({
+                    type: 'like',
+                    tweetId: like.tweetId,
+                    fullText: like.fullText,
+                    expandedUrl: like.expandedUrl,
+                }, { originalData: like });
+                count++;
             }
         }
         catch (error) {
-            console.warn(`Could not process likes: ${error}`);
+            this.logger.warn('Could not process likes', { error: String(error) });
         }
     }
+    /** The archive's likes, as it writes them: with no time, so since and until can't apply. */
+    async loadLikes(filePath) {
+        const content = await this.files.readText(filePath);
+        return this.parseTwitterJS(content, 'like')
+            .map(item => item.like)
+            .filter(Boolean);
+    }
     async determineCount() {
-        const config = this.config;
         try {
-            const likesContent = await readFile(`${config.input}/data/like.js`, 'utf-8');
+            const likesContent = await this.files.readText(`data/like.js`);
             const likesData = this.parseTwitterJS(likesContent, 'like');
             return likesData.length;
         }
@@ -56,4 +55,8 @@ export class TwitterArchiveLikesExtractor extends TwitterArchiveExtractor {
             return null;
         }
     }
+    /** The likes a run reads, counted alone: the archive doesn't say when any were made. */
+    async survey() {
+        return { likes: { count: (await this.loadLikes(`data/like.js`)).length } };
+    }
 }

package/dist/connectors/archive/TwitterArchiveTweetsExtractor.d.ts

@@ -1,5 +1,4 @@
-import { Record } from '@chronicle.app/etl';
-import { z } from 'zod';
+import { Record, type Survey, type SurveyOptions } from '@chronicle.app/etl';
 import TwitterTransformer from '../TwitterTransformer.js';
 import { TwitterArchiveExtractor } from './TwitterArchiveExtractor.js';
 export declare class TwitterArchiveTweetsExtractor extends TwitterArchiveExtractor<typeof TwitterArchiveTweetsExtractor> {
@@ -7,14 +6,14 @@ export declare class TwitterArchiveTweetsExtractor extends TwitterArchiveExtract
     static recordTypes: string[];
     static default: boolean;
     static defaultTransformer: typeof TwitterTransformer;
-    static schema: z.ZodObject<{
-        since: z.ZodOptional<z.ZodDate>;
-        until: z.ZodOptional<z.ZodDate>;
-        limit: z.ZodOptional<z.ZodNumber>;
+    static schema: import("zod").ZodObject<{
+        since: import("zod").ZodOptional<import("zod").ZodDate>;
+        until: import("zod").ZodOptional<import("zod").ZodDate>;
+        limit: import("zod").ZodOptional<import("zod").ZodNumber>;
     } & {
-        input: z.ZodString;
-        'include-group-dms': z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
-    }, "strip", z.ZodTypeAny, {
+        input: import("zod").ZodString;
+        'include-group-dms': import("zod").ZodDefault<import("zod").ZodOptional<import("zod").ZodBoolean>>;
+    }, "strip", import("zod").ZodTypeAny, {
         input: string;
         'include-group-dms': boolean;
         since?: Date | undefined;
@@ -31,6 +30,10 @@ export declare class TwitterArchiveTweetsExtractor extends TwitterArchiveExtract
     keyOf(record: Record): string | null;
     extract(): AsyncGenerator<Record>;
     private processTweets;
+    /** The archive's tweets within since and until, as it writes them. */
+    private loadTweets;
     determineCount(): Promise<number | null>;
+    /** The tweets a run reads and when they were posted. */
+    survey(options?: SurveyOptions): Promise<Survey>;
 }
 //# sourceMappingURL=TwitterArchiveTweetsExtractor.d.ts.map

package/dist/connectors/archive/TwitterArchiveTweetsExtractor.js

@@ -1,4 +1,3 @@
-import { readFile } from 'node:fs/promises';
 import TwitterTransformer from '../TwitterTransformer.js';
 import { TwitterArchiveExtractor } from './TwitterArchiveExtractor.js';
 export class TwitterArchiveTweetsExtractor extends TwitterArchiveExtractor {
@@ -12,11 +11,10 @@ export class TwitterArchiveTweetsExtractor extends TwitterArchiveExtractor {
         return record.data.id ?? null;
     }
     async *extract() {
-        const config = this.config;
         try {
             // Load account info first
-            await this.loadAccountInfoFrom(config.input);
-            yield* this.processTweets(`${config.input}/data/tweets.js`);
+            this.accountInfo = await this.loadAccountInfo();
+            yield* this.processTweets(`data/tweets.js`);
         }
         catch (error) {
             throw new Error(`Failed to process Twitter tweets: ${error}`);
@@ -24,39 +22,40 @@ export class TwitterArchiveTweetsExtractor extends TwitterArchiveExtractor {
     }
     async *processTweets(filePath) {
         try {
-            const content = await readFile(filePath, 'utf-8');
-            const jsData = this.parseTwitterJS(content, 'tweets');
             let count = 0;
-            for (const item of jsData) {
-                const { tweet } = item;
-                if (tweet && this.isWithinDateRange(new Date(tweet.created_at))) {
-                    if (this.shouldStopExtracting(count))
-                        break;
-                    yield this.createRecordWithArchiveContext({
-                        type: 'tweet',
-                        id: tweet.id_str,
-                        text: tweet.full_text || tweet.text,
-                        created_at: this.convertTwitterDateToISO(tweet.created_at),
-                        favorite_count: Number.parseInt(tweet.favorite_count) || 0,
-                        retweet_count: Number.parseInt(tweet.retweet_count) || 0,
-                        in_reply_to_status_id: tweet.in_reply_to_status_id_str,
-                        in_reply_to_user_id: tweet.in_reply_to_user_id_str,
-                        entities: tweet.entities,
-                        source: tweet.source,
-                        retweeted: tweet.retweeted,
-                    }, { originalData: tweet });
-                    count++;
-                }
+            for (const tweet of await this.loadTweets(filePath)) {
+                if (this.shouldStopExtracting(count))
+                    break;
+                yield this.createRecordWithArchiveContext({
+                    type: 'tweet',
+                    id: tweet.id_str,
+                    text: tweet.full_text || tweet.text,
+                    created_at: this.convertTwitterDateToISO(tweet.created_at),
+                    favorite_count: Number.parseInt(tweet.favorite_count) || 0,
+                    retweet_count: Number.parseInt(tweet.retweet_count) || 0,
+                    in_reply_to_status_id: tweet.in_reply_to_status_id_str,
+                    in_reply_to_user_id: tweet.in_reply_to_user_id_str,
+                    entities: tweet.entities,
+                    source: tweet.source,
+                    retweeted: tweet.retweeted,
+                }, { originalData: tweet });
+                count++;
             }
         }
         catch (error) {
-            console.warn(`Could not process tweets: ${error}`);
+            this.logger.warn('Could not process tweets', { error: String(error) });
         }
     }
+    /** The archive's tweets within since and until, as it writes them. */
+    async loadTweets(filePath) {
+        const content = await this.files.readText(filePath);
+        return this.parseTwitterJS(content, 'tweets')
+            .map(item => item.tweet)
+            .filter(tweet => tweet && this.isWithinDateRange(new Date(tweet.created_at)));
+    }
     async determineCount() {
-        const config = this.config;
         try {
-            const tweetsContent = await readFile(`${config.input}/data/tweets.js`, 'utf-8');
+            const tweetsContent = await this.files.readText(`data/tweets.js`);
             const tweetsData = this.parseTwitterJS(tweetsContent, 'tweets');
             return tweetsData.length;
         }
@@ -64,4 +63,11 @@ export class TwitterArchiveTweetsExtractor extends TwitterArchiveExtractor {
             return null;
         }
     }
+    /** The tweets a run reads and when they were posted. */
+    async survey(options = {}) {
+        const tweets = await this.loadTweets(`data/tweets.js`);
+        return {
+            tweets: this.surveyOf(tweets.map(tweet => new Date(tweet.created_at)), options),
+        };
+    }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chronicle.app/twitter",
-  "version": "0.3.0",
+  "version": "0.5.0",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "files": [
@@ -22,9 +22,9 @@
   "type": "module",
   "description": "Chronicle plugin for twitter",
   "devDependencies": {
-    "@chronicle.app/etl": "0.3.0",
-    "@chronicle.app/schema": "0.3.0",
-    "@chronicle.app/tsconfig": "0.3.0",
+    "@chronicle.app/etl": "0.5.0",
+    "@chronicle.app/schema": "0.5.0",
+    "@chronicle.app/tsconfig": "0.5.0",
     "@types/node": "^22.10.1",
     "typescript": "^5.7.2"
   },
@@ -61,6 +61,20 @@
         "strategies": {
           "archive": {
             "delivery": "export",
+            "layouts": [
+              {
+                "files": [
+                  "data/account.js",
+                  "data/tweets.js"
+                ],
+                "match": [
+                  {
+                    "path": "data/account.js",
+                    "head": "^window\\.YTD\\.account\\.part0"
+                  }
+                ]
+              }
+            ],
             "recordTypes": [
               "direct-messages",
               "group-direct-messages",