@chronicle.app/facebook 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/connectors/FacebookTransformer.js +3 -2
- package/dist/connectors/archive/FacebookArchiveExtractor.d.ts +11 -1
- package/dist/connectors/archive/FacebookArchiveExtractor.js +71 -59
- package/dist/connectors/archive/FacebookArchiveMessagesExtractor.d.ts +5 -1
- package/dist/connectors/archive/FacebookArchiveMessagesExtractor.js +33 -30
- package/dist/connectors/archive/FacebookArchiveReactionsExtractor.d.ts +5 -1
- package/dist/connectors/archive/FacebookArchiveReactionsExtractor.js +24 -30
- package/dist/connectors/archive/FacebookArchiveSearchHistoryExtractor.d.ts +5 -1
- package/dist/connectors/archive/FacebookArchiveSearchHistoryExtractor.js +20 -18
- package/package.json +12 -4
package/README.md
CHANGED
|
@@ -33,7 +33,7 @@ chronicle extract facebook \
|
|
|
33
33
|
|
|
34
34
|
The extractor supports the following options:
|
|
35
35
|
|
|
36
|
-
- `--input` (required):
|
|
36
|
+
- `--input` (required): The Facebook export, as a folder or zip
|
|
37
37
|
- `--type` (required): What to read: `messages`, `reactions`, `comments`, or `searches`
|
|
38
38
|
- `--since`: Extract records from this date onwards
|
|
39
39
|
- `--until`: Extract records up to this date
|
|
@@ -34,10 +34,11 @@ export default class FacebookTransformer extends ChronicleTransformer {
|
|
|
34
34
|
const messageAction = {
|
|
35
35
|
'@type': 'MessageAction',
|
|
36
36
|
timestamp: record.data.timestamp,
|
|
37
|
-
'@key': ['@type', 'source', 'timestamp', 'agent.name', '
|
|
37
|
+
'@key': ['@type', 'source', 'timestamp', 'agent.name', 'result.body'],
|
|
38
38
|
source: 'facebook',
|
|
39
39
|
agent: sender,
|
|
40
|
-
|
|
40
|
+
// Sending makes the message, so it's the action's result.
|
|
41
|
+
result: message,
|
|
41
42
|
};
|
|
42
43
|
actions.push(messageAction);
|
|
43
44
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { ArchiveExtractor } from '@chronicle.app/etl';
|
|
1
|
+
import { ArchiveExtractor, Record, type Survey, type SurveyOptions } from '@chronicle.app/etl';
|
|
2
2
|
import { z } from 'zod';
|
|
3
3
|
export declare abstract class FacebookArchiveExtractor extends ArchiveExtractor<typeof FacebookArchiveExtractor> {
|
|
4
4
|
static source: string;
|
|
@@ -20,8 +20,12 @@ export declare abstract class FacebookArchiveExtractor extends ArchiveExtractor<
|
|
|
20
20
|
until?: Date | undefined;
|
|
21
21
|
limit?: number | undefined;
|
|
22
22
|
}>;
|
|
23
|
+
protected readonly exportName = "Facebook export";
|
|
24
|
+
/** The export's segments, by their paths in it: `''` when the export is one. */
|
|
23
25
|
protected exportSegments: string[];
|
|
24
26
|
initialize(): Promise<void>;
|
|
27
|
+
/** Whether a file or folder is in the export. */
|
|
28
|
+
protected exists(path: string): Promise<boolean>;
|
|
25
29
|
protected discoverExportSegments(): Promise<void>;
|
|
26
30
|
protected loadAccountInfo(): Promise<any>;
|
|
27
31
|
protected iterateExportSegments(): AsyncGenerator<{
|
|
@@ -29,6 +33,12 @@ export declare abstract class FacebookArchiveExtractor extends ArchiveExtractor<
|
|
|
29
33
|
facebookDir: string;
|
|
30
34
|
fullPath: string;
|
|
31
35
|
}>;
|
|
36
|
+
/**
|
|
37
|
+
* A survey of records as a run reads them: each counted under its record
|
|
38
|
+
* type and dated by its `timestamp`, as the transformer dates its action.
|
|
39
|
+
* The export has no index, so every file is read, and only the tallies are kept.
|
|
40
|
+
*/
|
|
41
|
+
protected surveyOf(records: AsyncIterable<Record>, options: SurveyOptions, each?: (record: Record) => void): Promise<Survey>;
|
|
32
42
|
protected readFacebookJson(filePath: string): Promise<any>;
|
|
33
43
|
protected convertFacebookTimestamp(timestamp: number): string;
|
|
34
44
|
protected fixFacebookTextEncoding(text: string): string;
|
|
@@ -1,54 +1,49 @@
|
|
|
1
|
-
import { ArchiveExtractor, Extractor } from '@chronicle.app/etl';
|
|
2
|
-
import {
|
|
3
|
-
import { join } from 'node:path';
|
|
1
|
+
import { ArchiveExtractor, Extractor, countMonth, } from '@chronicle.app/etl';
|
|
2
|
+
import { posix } from 'node:path';
|
|
4
3
|
import { z } from 'zod';
|
|
4
|
+
// The folders a Facebook export starts with, beside its other sections.
|
|
5
|
+
const SECTIONS = new Set(['personal_information/', 'your_facebook_activity/']);
|
|
5
6
|
export class FacebookArchiveExtractor extends ArchiveExtractor {
|
|
6
7
|
static source = 'facebook';
|
|
7
8
|
static strategy = 'archive';
|
|
8
9
|
static schema = Extractor.schema.extend({
|
|
9
|
-
input: z.string().describe('
|
|
10
|
+
input: z.string().describe('The Facebook export, as a folder or zip'),
|
|
10
11
|
});
|
|
12
|
+
exportName = 'Facebook export';
|
|
13
|
+
/** The export's segments, by their paths in it: `''` when the export is one. */
|
|
11
14
|
exportSegments = [];
|
|
12
15
|
async initialize() {
|
|
13
16
|
await this.discoverExportSegments();
|
|
14
17
|
this.accountInfo = await this.loadAccountInfo();
|
|
15
18
|
}
|
|
19
|
+
/** Whether a file or folder is in the export. */
|
|
20
|
+
async exists(path) {
|
|
21
|
+
return (await this.files.has(path)) || (await this.files.children(path)).length > 0;
|
|
22
|
+
}
|
|
16
23
|
async discoverExportSegments() {
|
|
17
|
-
const
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
//
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
if (hasDirectStructure) {
|
|
24
|
-
// New format: direct structure
|
|
25
|
-
this.exportSegments = [config.input];
|
|
26
|
-
// Detected new Facebook export format (direct structure)
|
|
27
|
-
}
|
|
28
|
-
else {
|
|
29
|
-
// Old format: Find all export segment directories (meta-* or facebook-*)
|
|
30
|
-
this.exportSegments = entries
|
|
31
|
-
.filter(entry => entry.isDirectory())
|
|
32
|
-
.map(entry => entry.name)
|
|
33
|
-
.filter(name => name.startsWith('meta-') || name.startsWith('facebook-'))
|
|
34
|
-
.map(name => join(config.input, name));
|
|
35
|
-
// Discovered export segments (legacy format)
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
catch {
|
|
39
|
-
// Could not discover export segments
|
|
40
|
-
this.exportSegments = [config.input]; // Fallback to input path
|
|
24
|
+
const entries = await this.files.children('');
|
|
25
|
+
// Check if this is a new format export (direct structure)
|
|
26
|
+
if (entries.some(entry => SECTIONS.has(entry))) {
|
|
27
|
+
// New format: direct structure
|
|
28
|
+
this.exportSegments = [''];
|
|
29
|
+
return;
|
|
41
30
|
}
|
|
31
|
+
// Old format: Find all export segment directories (meta-* or facebook-*)
|
|
32
|
+
const segments = entries
|
|
33
|
+
.filter(entry => entry.endsWith('/'))
|
|
34
|
+
.map(entry => entry.slice(0, -1))
|
|
35
|
+
.filter(name => name.startsWith('meta-') || name.startsWith('facebook-'));
|
|
36
|
+
// Without any, fall back to the export's root
|
|
37
|
+
this.exportSegments = segments.length > 0 ? segments : [''];
|
|
42
38
|
}
|
|
43
39
|
async loadAccountInfo() {
|
|
44
40
|
// Try to find account info in any of the segments
|
|
45
41
|
for (const segmentPath of this.exportSegments) {
|
|
46
42
|
try {
|
|
47
43
|
// Check for new format (direct structure)
|
|
48
|
-
const directProfilePath = join(segmentPath, 'personal_information', 'profile_information', 'profile_information.json');
|
|
44
|
+
const directProfilePath = posix.join(segmentPath, 'personal_information', 'profile_information', 'profile_information.json');
|
|
49
45
|
try {
|
|
50
|
-
const
|
|
51
|
-
const data = JSON.parse(content);
|
|
46
|
+
const data = await this.readFacebookJson(directProfilePath);
|
|
52
47
|
return {
|
|
53
48
|
name: data.profile_v2?.name?.full_name || data.profile_v2?.name?.value || 'unknown',
|
|
54
49
|
email: data.profile_v2?.emails?.emails?.[0] ||
|
|
@@ -64,15 +59,13 @@ export class FacebookArchiveExtractor extends ArchiveExtractor {
|
|
|
64
59
|
// Not new format, try legacy format
|
|
65
60
|
}
|
|
66
61
|
// Legacy format: look for facebook-* subdirectories
|
|
67
|
-
const
|
|
68
|
-
|
|
69
|
-
.
|
|
70
|
-
.map(entry => entry.name);
|
|
62
|
+
const facebookDirs = (await this.files.children(segmentPath))
|
|
63
|
+
.filter(entry => entry.endsWith('/') && entry.startsWith('facebook-'))
|
|
64
|
+
.map(entry => entry.slice(0, -1));
|
|
71
65
|
for (const fbDir of facebookDirs) {
|
|
72
66
|
try {
|
|
73
|
-
const personalInfoPath = join(segmentPath, fbDir, 'personal_information', 'profile_information', 'profile_information.json');
|
|
74
|
-
const
|
|
75
|
-
const data = JSON.parse(content);
|
|
67
|
+
const personalInfoPath = posix.join(segmentPath, fbDir, 'personal_information', 'profile_information', 'profile_information.json');
|
|
68
|
+
const data = await this.readFacebookJson(personalInfoPath);
|
|
76
69
|
return {
|
|
77
70
|
name: data.profile_v2?.name?.full_name || data.profile_v2?.name?.value || 'unknown',
|
|
78
71
|
email: data.profile_v2?.emails?.emails?.[0] ||
|
|
@@ -98,31 +91,50 @@ export class FacebookArchiveExtractor extends ArchiveExtractor {
|
|
|
98
91
|
}
|
|
99
92
|
async *iterateExportSegments() {
|
|
100
93
|
for (const segmentPath of this.exportSegments) {
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
if (hasDirectStructure) {
|
|
107
|
-
// New format: yield the segment path directly
|
|
108
|
-
yield { segmentPath, facebookDir: '', fullPath: segmentPath };
|
|
109
|
-
}
|
|
110
|
-
else {
|
|
111
|
-
// Legacy format: look for facebook-* subdirectories
|
|
112
|
-
const facebookDirs = entries
|
|
113
|
-
.filter(entry => entry.isDirectory() && entry.name.startsWith('facebook-'))
|
|
114
|
-
.map(entry => entry.name);
|
|
115
|
-
for (const facebookDir of facebookDirs) {
|
|
116
|
-
const fullPath = join(segmentPath, facebookDir);
|
|
117
|
-
yield { segmentPath, facebookDir, fullPath };
|
|
118
|
-
}
|
|
119
|
-
}
|
|
94
|
+
// Check if this is new format (direct structure)
|
|
95
|
+
const entries = await this.files.children(segmentPath);
|
|
96
|
+
if (entries.some(entry => SECTIONS.has(entry))) {
|
|
97
|
+
// New format: yield the segment path directly
|
|
98
|
+
yield { segmentPath, facebookDir: '', fullPath: segmentPath };
|
|
120
99
|
}
|
|
121
|
-
|
|
122
|
-
//
|
|
100
|
+
else {
|
|
101
|
+
// Legacy format: look for facebook-* subdirectories
|
|
102
|
+
const facebookDirs = entries
|
|
103
|
+
.filter(entry => entry.endsWith('/') && entry.startsWith('facebook-'))
|
|
104
|
+
.map(entry => entry.slice(0, -1));
|
|
105
|
+
for (const facebookDir of facebookDirs) {
|
|
106
|
+
const fullPath = posix.join(segmentPath, facebookDir);
|
|
107
|
+
yield { segmentPath, facebookDir, fullPath };
|
|
108
|
+
}
|
|
123
109
|
}
|
|
124
110
|
}
|
|
125
111
|
}
|
|
112
|
+
/**
|
|
113
|
+
* A survey of records as a run reads them: each counted under its record
|
|
114
|
+
* type and dated by its `timestamp`, as the transformer dates its action.
|
|
115
|
+
* The export has no index, so every file is read, and only the tallies are kept.
|
|
116
|
+
*/
|
|
117
|
+
async surveyOf(records, options, each) {
|
|
118
|
+
const kind = () => ({ count: 0, ...(options.byMonth && { byMonth: {} }) });
|
|
119
|
+
const { recordTypes } = this.constructor;
|
|
120
|
+
const survey = Object.fromEntries(recordTypes.map(recordType => [recordType, kind()]));
|
|
121
|
+
for await (const record of records) {
|
|
122
|
+
each?.(record);
|
|
123
|
+
// A record without a type is its extractor's first, as `createRecord` gives it.
|
|
124
|
+
const counted = survey[record.extraction.recordType ?? recordTypes[0]];
|
|
125
|
+
counted.count++;
|
|
126
|
+
if (!record.data.timestamp)
|
|
127
|
+
continue;
|
|
128
|
+
const time = new Date(record.data.timestamp);
|
|
129
|
+
if (!counted.earliest || time < counted.earliest)
|
|
130
|
+
counted.earliest = time;
|
|
131
|
+
if (!counted.latest || time > counted.latest)
|
|
132
|
+
counted.latest = time;
|
|
133
|
+
if (counted.byMonth)
|
|
134
|
+
countMonth(counted.byMonth, time);
|
|
135
|
+
}
|
|
136
|
+
return survey;
|
|
137
|
+
}
|
|
126
138
|
async readFacebookJson(filePath) {
|
|
127
139
|
return this.readArchiveJson(filePath);
|
|
128
140
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Record } from '@chronicle.app/etl';
|
|
1
|
+
import { Record, type Survey, type SurveyOptions } from '@chronicle.app/etl';
|
|
2
2
|
import { FacebookArchiveExtractor } from './FacebookArchiveExtractor.js';
|
|
3
3
|
import FacebookTransformer from '../FacebookTransformer.js';
|
|
4
4
|
export declare class FacebookArchiveMessagesExtractor extends FacebookArchiveExtractor {
|
|
@@ -7,6 +7,10 @@ export declare class FacebookArchiveMessagesExtractor extends FacebookArchiveExt
|
|
|
7
7
|
static default: boolean;
|
|
8
8
|
static defaultTransformer: typeof FacebookTransformer;
|
|
9
9
|
extract(): AsyncGenerator<Record>;
|
|
10
|
+
/** The messages a run reads and when they were sent, with how many conversations they're in. */
|
|
11
|
+
survey(options?: SurveyOptions): Promise<Survey>;
|
|
12
|
+
/** Every message in the export, in the order a run reads them, before its limit. */
|
|
13
|
+
private messages;
|
|
10
14
|
private extractMessagesFromSegment;
|
|
11
15
|
private extractFromMessageFolder;
|
|
12
16
|
private extractMessagesFromConversation;
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { join } from 'node:path';
|
|
1
|
+
import { posix } from 'node:path';
|
|
3
2
|
import { FacebookArchiveExtractor } from './FacebookArchiveExtractor.js';
|
|
4
3
|
import FacebookTransformer from '../FacebookTransformer.js';
|
|
5
4
|
export class FacebookArchiveMessagesExtractor extends FacebookArchiveExtractor {
|
|
@@ -8,28 +7,36 @@ export class FacebookArchiveMessagesExtractor extends FacebookArchiveExtractor {
|
|
|
8
7
|
static default = true;
|
|
9
8
|
static defaultTransformer = FacebookTransformer;
|
|
10
9
|
async *extract() {
|
|
10
|
+
let recordCount = 0;
|
|
11
|
+
for await (const record of this.messages()) {
|
|
12
|
+
yield record;
|
|
13
|
+
recordCount++;
|
|
14
|
+
if (this.shouldStopExtracting(recordCount)) {
|
|
15
|
+
return;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
/** The messages a run reads and when they were sent, with how many conversations they're in. */
|
|
20
|
+
async survey(options = {}) {
|
|
21
|
+
const conversations = new Set();
|
|
22
|
+
const survey = await this.surveyOf(this.messages(), options, record => conversations.add(record.context.conversationId));
|
|
23
|
+
survey.messages.measures = [
|
|
24
|
+
{ key: 'conversations', label: 'conversations', value: conversations.size },
|
|
25
|
+
];
|
|
26
|
+
return survey;
|
|
27
|
+
}
|
|
28
|
+
/** Every message in the export, in the order a run reads them, before its limit. */
|
|
29
|
+
async *messages() {
|
|
11
30
|
await this.initialize();
|
|
12
31
|
const processedConversations = new Set();
|
|
13
|
-
let recordCount = 0;
|
|
14
32
|
for await (const { fullPath } of this.iterateExportSegments()) {
|
|
15
|
-
|
|
16
|
-
yield record;
|
|
17
|
-
recordCount++;
|
|
18
|
-
if (this.shouldStopExtracting(recordCount)) {
|
|
19
|
-
return;
|
|
20
|
-
}
|
|
21
|
-
}
|
|
33
|
+
yield* this.extractMessagesFromSegment(fullPath, processedConversations);
|
|
22
34
|
}
|
|
23
35
|
}
|
|
24
36
|
async *extractMessagesFromSegment(segmentPath, processedConversations) {
|
|
25
|
-
const messagesPath = join(segmentPath, 'your_facebook_activity', 'messages');
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
}
|
|
29
|
-
catch {
|
|
30
|
-
// No messages directory found
|
|
31
|
-
return;
|
|
32
|
-
}
|
|
37
|
+
const messagesPath = posix.join(segmentPath, 'your_facebook_activity', 'messages');
|
|
38
|
+
if (!(await this.exists(messagesPath)))
|
|
39
|
+
return; // No messages directory found
|
|
33
40
|
// Process different message types
|
|
34
41
|
yield* this.extractFromMessageFolder(messagesPath, 'inbox', processedConversations);
|
|
35
42
|
yield* this.extractFromMessageFolder(messagesPath, 'archived_threads', processedConversations);
|
|
@@ -37,24 +44,20 @@ export class FacebookArchiveMessagesExtractor extends FacebookArchiveExtractor {
|
|
|
37
44
|
yield* this.extractFromMessageFolder(messagesPath, 'filtered_threads', processedConversations);
|
|
38
45
|
}
|
|
39
46
|
async *extractFromMessageFolder(messagesPath, folderName, processedConversations) {
|
|
40
|
-
const folderPath = join(messagesPath, folderName);
|
|
41
|
-
|
|
42
|
-
await stat(folderPath);
|
|
43
|
-
}
|
|
44
|
-
catch {
|
|
47
|
+
const folderPath = posix.join(messagesPath, folderName);
|
|
48
|
+
if (!(await this.exists(folderPath)))
|
|
45
49
|
return; // Folder doesn't exist in this segment
|
|
46
|
-
}
|
|
47
50
|
try {
|
|
48
|
-
const conversations = await
|
|
51
|
+
const conversations = await this.files.children(folderPath);
|
|
49
52
|
for (const conversation of conversations) {
|
|
50
|
-
if (!conversation.
|
|
53
|
+
if (!conversation.endsWith('/'))
|
|
51
54
|
continue;
|
|
52
|
-
const conversationId = conversation.
|
|
55
|
+
const conversationId = conversation.slice(0, -1);
|
|
53
56
|
// Skip if already processed (avoid duplicates across segments)
|
|
54
57
|
if (processedConversations.has(conversationId)) {
|
|
55
58
|
continue;
|
|
56
59
|
}
|
|
57
|
-
const conversationPath = join(folderPath, conversationId);
|
|
60
|
+
const conversationPath = posix.join(folderPath, conversationId);
|
|
58
61
|
try {
|
|
59
62
|
yield* this.extractMessagesFromConversation(conversationPath, conversationId, folderName);
|
|
60
63
|
processedConversations.add(conversationId);
|
|
@@ -70,7 +73,7 @@ export class FacebookArchiveMessagesExtractor extends FacebookArchiveExtractor {
|
|
|
70
73
|
}
|
|
71
74
|
async *extractMessagesFromConversation(conversationPath, conversationId, threadType) {
|
|
72
75
|
try {
|
|
73
|
-
const files = await
|
|
76
|
+
const files = await this.files.children(conversationPath);
|
|
74
77
|
const messageFiles = files.filter(file => file.startsWith('message_') && file.endsWith('.json'));
|
|
75
78
|
if (messageFiles.length === 0) {
|
|
76
79
|
// No message files found in conversation
|
|
@@ -83,7 +86,7 @@ export class FacebookArchiveMessagesExtractor extends FacebookArchiveExtractor {
|
|
|
83
86
|
return aNum - bNum;
|
|
84
87
|
});
|
|
85
88
|
for (const messageFile of messageFiles) {
|
|
86
|
-
const filePath = join(conversationPath, messageFile);
|
|
89
|
+
const filePath = posix.join(conversationPath, messageFile);
|
|
87
90
|
try {
|
|
88
91
|
const data = await this.readFacebookJson(filePath);
|
|
89
92
|
yield* this.processConversationData(data, conversationId, threadType);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Record } from '@chronicle.app/etl';
|
|
1
|
+
import { Record, type Survey, type SurveyOptions } from '@chronicle.app/etl';
|
|
2
2
|
import { FacebookArchiveExtractor } from './FacebookArchiveExtractor.js';
|
|
3
3
|
import FacebookTransformer from '../FacebookTransformer.js';
|
|
4
4
|
export declare class FacebookArchiveReactionsExtractor extends FacebookArchiveExtractor {
|
|
@@ -13,6 +13,10 @@ export declare class FacebookArchiveReactionsExtractor extends FacebookArchiveEx
|
|
|
13
13
|
*/
|
|
14
14
|
keyOf(record: Record): string | null;
|
|
15
15
|
extract(): AsyncGenerator<Record>;
|
|
16
|
+
/** The reactions and comments a run reads, and when each was made. */
|
|
17
|
+
survey(options?: SurveyOptions): Promise<Survey>;
|
|
18
|
+
/** Every reaction and comment in the export, in the order a run reads them, before its limit. */
|
|
19
|
+
private reactionsAndComments;
|
|
16
20
|
private extractReactionsFromSegment;
|
|
17
21
|
private extractFromReactionsFile;
|
|
18
22
|
private extractFromCommentsFile;
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { join } from 'node:path';
|
|
1
|
+
import { posix } from 'node:path';
|
|
3
2
|
import { FacebookArchiveExtractor } from './FacebookArchiveExtractor.js';
|
|
4
3
|
import FacebookTransformer from '../FacebookTransformer.js';
|
|
5
4
|
export class FacebookArchiveReactionsExtractor extends FacebookArchiveExtractor {
|
|
@@ -16,41 +15,40 @@ export class FacebookArchiveReactionsExtractor extends FacebookArchiveExtractor
|
|
|
16
15
|
return record.data.fbid ?? null;
|
|
17
16
|
}
|
|
18
17
|
async *extract() {
|
|
18
|
+
let recordCount = 0;
|
|
19
|
+
for await (const record of this.reactionsAndComments()) {
|
|
20
|
+
yield record;
|
|
21
|
+
recordCount++;
|
|
22
|
+
if (this.shouldStopExtracting(recordCount)) {
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
/** The reactions and comments a run reads, and when each was made. */
|
|
28
|
+
async survey(options = {}) {
|
|
29
|
+
return this.surveyOf(this.reactionsAndComments(), options);
|
|
30
|
+
}
|
|
31
|
+
/** Every reaction and comment in the export, in the order a run reads them, before its limit. */
|
|
32
|
+
async *reactionsAndComments() {
|
|
19
33
|
await this.initialize();
|
|
20
34
|
const processedReactions = new Set();
|
|
21
|
-
let recordCount = 0;
|
|
22
35
|
for await (const { fullPath } of this.iterateExportSegments()) {
|
|
23
|
-
|
|
24
|
-
yield record;
|
|
25
|
-
recordCount++;
|
|
26
|
-
if (this.shouldStopExtracting(recordCount)) {
|
|
27
|
-
return;
|
|
28
|
-
}
|
|
29
|
-
}
|
|
36
|
+
yield* this.extractReactionsFromSegment(fullPath, processedReactions);
|
|
30
37
|
}
|
|
31
38
|
}
|
|
32
39
|
async *extractReactionsFromSegment(segmentPath, processedReactions) {
|
|
33
|
-
const reactionsPath = join(segmentPath, 'your_facebook_activity', 'comments_and_reactions');
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
}
|
|
37
|
-
catch {
|
|
38
|
-
// No reactions directory found
|
|
39
|
-
return;
|
|
40
|
-
}
|
|
40
|
+
const reactionsPath = posix.join(segmentPath, 'your_facebook_activity', 'comments_and_reactions');
|
|
41
|
+
if (!(await this.exists(reactionsPath)))
|
|
42
|
+
return; // No reactions directory found
|
|
41
43
|
// Process different reaction/comment files
|
|
42
44
|
yield* this.extractFromReactionsFile(reactionsPath, 'likes_and_reactions.json', processedReactions);
|
|
43
45
|
yield* this.extractFromReactionsFile(reactionsPath, 'likes_and_reactions_1.json', processedReactions);
|
|
44
46
|
yield* this.extractFromCommentsFile(reactionsPath, 'comments.json', processedReactions);
|
|
45
47
|
}
|
|
46
48
|
async *extractFromReactionsFile(reactionsPath, fileName, processedReactions) {
|
|
47
|
-
const filePath = join(reactionsPath, fileName);
|
|
48
|
-
|
|
49
|
-
await stat(filePath);
|
|
50
|
-
}
|
|
51
|
-
catch {
|
|
49
|
+
const filePath = posix.join(reactionsPath, fileName);
|
|
50
|
+
if (!(await this.exists(filePath)))
|
|
52
51
|
return; // File doesn't exist
|
|
53
|
-
}
|
|
54
52
|
try {
|
|
55
53
|
const data = await this.readFacebookJson(filePath);
|
|
56
54
|
if (Array.isArray(data)) {
|
|
@@ -68,13 +66,9 @@ export class FacebookArchiveReactionsExtractor extends FacebookArchiveExtractor
|
|
|
68
66
|
}
|
|
69
67
|
}
|
|
70
68
|
async *extractFromCommentsFile(reactionsPath, fileName, processedReactions) {
|
|
71
|
-
const filePath = join(reactionsPath, fileName);
|
|
72
|
-
|
|
73
|
-
await stat(filePath);
|
|
74
|
-
}
|
|
75
|
-
catch {
|
|
69
|
+
const filePath = posix.join(reactionsPath, fileName);
|
|
70
|
+
if (!(await this.exists(filePath)))
|
|
76
71
|
return; // File doesn't exist
|
|
77
|
-
}
|
|
78
72
|
try {
|
|
79
73
|
const data = await this.readFacebookJson(filePath);
|
|
80
74
|
if (data.comments_v2 && Array.isArray(data.comments_v2)) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Record } from '@chronicle.app/etl';
|
|
1
|
+
import { Record, type Survey, type SurveyOptions } from '@chronicle.app/etl';
|
|
2
2
|
import { FacebookArchiveExtractor } from './FacebookArchiveExtractor.js';
|
|
3
3
|
import FacebookTransformer from '../FacebookTransformer.js';
|
|
4
4
|
export declare class FacebookArchiveSearchHistoryExtractor extends FacebookArchiveExtractor {
|
|
@@ -7,6 +7,10 @@ export declare class FacebookArchiveSearchHistoryExtractor extends FacebookArchi
|
|
|
7
7
|
static default: boolean;
|
|
8
8
|
static defaultTransformer: typeof FacebookTransformer;
|
|
9
9
|
extract(): AsyncGenerator<Record>;
|
|
10
|
+
/** The searches a run reads, and when each was made. */
|
|
11
|
+
survey(options?: SurveyOptions): Promise<Survey>;
|
|
12
|
+
/** Every search in the export, in the order a run reads them, before its limit. */
|
|
13
|
+
private searches;
|
|
10
14
|
private extractSearchHistoryFromSegment;
|
|
11
15
|
private processSearchHistoryData;
|
|
12
16
|
private parseSearchData;
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { join } from 'node:path';
|
|
1
|
+
import { posix } from 'node:path';
|
|
3
2
|
import { FacebookArchiveExtractor } from './FacebookArchiveExtractor.js';
|
|
4
3
|
import FacebookTransformer from '../FacebookTransformer.js';
|
|
5
4
|
export class FacebookArchiveSearchHistoryExtractor extends FacebookArchiveExtractor {
|
|
@@ -8,28 +7,31 @@ export class FacebookArchiveSearchHistoryExtractor extends FacebookArchiveExtrac
|
|
|
8
7
|
static default = false;
|
|
9
8
|
static defaultTransformer = FacebookTransformer;
|
|
10
9
|
async *extract() {
|
|
10
|
+
let recordCount = 0;
|
|
11
|
+
for await (const record of this.searches()) {
|
|
12
|
+
yield record;
|
|
13
|
+
recordCount++;
|
|
14
|
+
if (this.shouldStopExtracting(recordCount)) {
|
|
15
|
+
return;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
/** The searches a run reads, and when each was made. */
|
|
20
|
+
async survey(options = {}) {
|
|
21
|
+
return this.surveyOf(this.searches(), options);
|
|
22
|
+
}
|
|
23
|
+
/** Every search in the export, in the order a run reads them, before its limit. */
|
|
24
|
+
async *searches() {
|
|
11
25
|
await this.initialize();
|
|
12
26
|
const processedSearches = new Set();
|
|
13
|
-
let recordCount = 0;
|
|
14
27
|
for await (const { fullPath } of this.iterateExportSegments()) {
|
|
15
|
-
|
|
16
|
-
yield record;
|
|
17
|
-
recordCount++;
|
|
18
|
-
if (this.shouldStopExtracting(recordCount)) {
|
|
19
|
-
return;
|
|
20
|
-
}
|
|
21
|
-
}
|
|
28
|
+
yield* this.extractSearchHistoryFromSegment(fullPath, processedSearches);
|
|
22
29
|
}
|
|
23
30
|
}
|
|
24
31
|
async *extractSearchHistoryFromSegment(segmentPath, processedSearches) {
|
|
25
|
-
const searchHistoryPath = join(segmentPath, 'logged_information', 'search', 'your_search_history.json');
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
}
|
|
29
|
-
catch {
|
|
30
|
-
// No search history file found
|
|
31
|
-
return;
|
|
32
|
-
}
|
|
32
|
+
const searchHistoryPath = posix.join(segmentPath, 'logged_information', 'search', 'your_search_history.json');
|
|
33
|
+
if (!(await this.exists(searchHistoryPath)))
|
|
34
|
+
return; // No search history file found
|
|
33
35
|
try {
|
|
34
36
|
const data = await this.readFacebookJson(searchHistoryPath);
|
|
35
37
|
yield* this.processSearchHistoryData(data, processedSearches);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@chronicle.app/facebook",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.0",
|
|
4
4
|
"main": "./dist/index.js",
|
|
5
5
|
"types": "./dist/index.d.ts",
|
|
6
6
|
"files": [
|
|
@@ -22,9 +22,9 @@
|
|
|
22
22
|
"type": "module",
|
|
23
23
|
"description": "Chronicle plugin for Chronicle plugin for facebook",
|
|
24
24
|
"devDependencies": {
|
|
25
|
-
"@chronicle.app/etl": "0.
|
|
26
|
-
"@chronicle.app/schema": "0.
|
|
27
|
-
"@chronicle.app/tsconfig": "0.
|
|
25
|
+
"@chronicle.app/etl": "0.5.0",
|
|
26
|
+
"@chronicle.app/schema": "0.5.0",
|
|
27
|
+
"@chronicle.app/tsconfig": "0.5.0",
|
|
28
28
|
"@types/node": "^22.10.1",
|
|
29
29
|
"typescript": "^5.7.2"
|
|
30
30
|
},
|
|
@@ -61,6 +61,14 @@
|
|
|
61
61
|
"strategies": {
|
|
62
62
|
"archive": {
|
|
63
63
|
"delivery": "export",
|
|
64
|
+
"layouts": [
|
|
65
|
+
{
|
|
66
|
+
"files": [
|
|
67
|
+
"personal_information/profile_information/profile_information.json",
|
|
68
|
+
"your_facebook_activity"
|
|
69
|
+
]
|
|
70
|
+
}
|
|
71
|
+
],
|
|
64
72
|
"recordTypes": [
|
|
65
73
|
"messages",
|
|
66
74
|
"reactions",
|