@chronicle.app/safari 0.4.0 → 0.5.0

package/README.md

@@ -4,6 +4,12 @@ Read Safari's `History.db` with `SafariExtractor`, then emit one `ViewAction` pe
 
 Records are visits, newest first, with redirect hops left out. `since`/`until` are exclusive visit-time bounds, and `limit: 0` means unlimited.
 
-Each `ViewAction` is keyed by its timestamp. The visited page is an `Entity` keyed by its URL and named after the page title, or the URL when there is none. The agent is the iCloud account owner from `@chronicle.app/icloud`, omitted when no account can be read. An explicit `account` object (or null) describes exported data without a host account lookup.
+Each `ViewAction` is keyed by its timestamp. The visited page is an `Entity` keyed by its URL and named after the page title, or the URL when there is none. The agent is the iCloud account owner from `@chronicle.app/apple`, omitted when no account can be read. An explicit `account` object (or null) describes exported data without a host account lookup.
 
-Tests use a synthetic history database and never read the host's browsing history or iCloud account.
+## Bookmarks
+
+`SafariBookmarksExtractor` reads the `bookmarks` record type from Safari's `Bookmarks.plist`, which also needs Full Disk Access. It's the one in the same folder as the `History.db` that `input` names, so `~/Library/Safari/Bookmarks.plist` by default, and a copy of Safari's folder is read as one. A `History.db` with none beside it has no bookmarks. `bookmarksFile` names another. Each bookmark, favorite, and Reading List item becomes a `BookmarkAction` whose `object` is the page and whose `target` is the folder it's in now, a `Collection`: Favorites, Bookmarks Menu, Reading List, or a folder of your own, which `isPartOf` the folder it's in. A bookmark outside any folder has no target. The file holds only what's saved now, so a run is a snapshot, asserted when the file last changed.
+
+A bookmark is keyed by its iCloud record name, which is the same on each device, or by its ID on this Mac when it doesn't sync. A folder is keyed by its ID within the iCloud account that holds it (`inAccount`), since Safari names its own folders, such as `Favorites Bar`, the same in every account. Each is dated by when it was added. Safari records that for every Reading List item, but only for bookmarks it saved recently, so older bookmarks are left out. Records are newest first, and `since`/`until` are exclusive bounds on the date.
+
+Tests use a synthetic history database and bookmarks file and never read the host's browsing history, bookmarks, or iCloud account.

package/dist/connectors/SafariBookmarksExtractor.d.ts

@@ -0,0 +1,105 @@
+import { Extractor, Record, type Survey, type SurveyOptions } from '@chronicle.app/etl';
+import { z } from 'zod';
+import SafariTransformer from './SafariTransformer.js';
+/** A folder a bookmark is in. */
+export interface BookmarkFolder {
+    id: string;
+    name: string;
+}
+/** A bookmark, a favorite, or a Reading List item. */
+export interface Bookmark {
+    id: string;
+    url: string;
+    title: string | null;
+    /** When it was saved (ISO). */
+    addedAt: string;
+    /** The folders it's in, outermost first. */
+    folders: BookmarkFolder[];
+}
+export declare class SafariBookmarksExtractor extends Extractor<typeof SafariBookmarksExtractor> {
+    static source: string;
+    static delivery: "local";
+    static strategy: string;
+    static description: string;
+    static default: boolean;
+    static recordTypes: string[];
+    static temporality: "snapshot";
+    static schema: z.ZodObject<{
+        since: z.ZodOptional<z.ZodDate>;
+        until: z.ZodOptional<z.ZodDate>;
+        limit: z.ZodOptional<z.ZodNumber>;
+    } & {
+        input: z.ZodDefault<z.ZodString>;
+        bookmarksFile: z.ZodOptional<z.ZodString>;
+        account: z.ZodOptional<z.ZodNullable<z.ZodObject<{
+            accountID: z.ZodString;
+            email: z.ZodString;
+            displayName: z.ZodString;
+            dsid: z.ZodOptional<z.ZodString>;
+        }, "strip", z.ZodTypeAny, {
+            accountID: string;
+            email: string;
+            displayName: string;
+            dsid?: string | undefined;
+        }, {
+            accountID: string;
+            email: string;
+            displayName: string;
+            dsid?: string | undefined;
+        }>>>;
+    }, "strip", z.ZodTypeAny, {
+        input: string;
+        bookmarksFile?: string | undefined;
+        account?: {
+            accountID: string;
+            email: string;
+            displayName: string;
+            dsid?: string | undefined;
+        } | null | undefined;
+        since?: Date | undefined;
+        until?: Date | undefined;
+        limit?: number | undefined;
+    }, {
+        input?: string | undefined;
+        bookmarksFile?: string | undefined;
+        account?: {
+            accountID: string;
+            email: string;
+            displayName: string;
+            dsid?: string | undefined;
+        } | null | undefined;
+        since?: Date | undefined;
+        until?: Date | undefined;
+        limit?: number | undefined;
+    }>;
+    static defaultTransformer: typeof SafariTransformer;
+    private bookmarks;
+    private changed;
+    /**
+     * Read the bookmarks file, by default the one in the folder of the
+     * History.db a run reads, so a copy of Safari's folder is read as one. A
+     * History.db without one beside it has no bookmarks to read.
+     */
+    setup(): Promise<void>;
+    /**
+     * When the bookmarks last changed, not when the run started, so a re-read of
+     * unchanged bookmarks comes out the same.
+     */
+    protected asOfTime(): Date;
+    /** The bookmark's CloudKit record name, the same on each device, or its ID on this Mac. */
+    keyOf(record: Record): string | null;
+    occurredAt(record: Record): Date;
+    extract(): AsyncGenerator<Record>;
+    /** The bookmarks a run reads and when, from the file setup() read. */
+    survey(options?: SurveyOptions): Promise<Survey>;
+    /** The bookmarks within since/until, newest first. */
+    private inRange;
+    /**
+     * Every bookmark in the file that Safari dated: each Reading List item, and
+     * each bookmark Safari saved since it began dating them, but not those from
+     * before.
+     */
+    private read;
+    private idOf;
+}
+//# sourceMappingURL=SafariBookmarksExtractor.d.ts.map

package/dist/connectors/SafariBookmarksExtractor.js

@@ -0,0 +1,149 @@
+import { readFile, stat } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { Extractor, countMonth, fileError, } from '@chronicle.app/etl';
+import { getICloudAccount, parseBinaryPlist, Uid } from '@chronicle.app/apple';
+import { z } from 'zod';
+import { accountOption, historyInput } from './SafariExtractor.js';
+import SafariTransformer from './SafariTransformer.js';
+// The folders Safari makes, by the names its sidebar gives them.
+const FOLDER_NAMES = {
+    BookmarksBar: 'Favorites',
+    BookmarksMenu: 'Bookmarks Menu',
+    'com.apple.ReadingList': 'Reading List',
+};
+const isNode = (value) => Boolean(value) &&
+    typeof value === 'object' &&
+    !Array.isArray(value) &&
+    !(value instanceof Date) &&
+    !(value instanceof Uint8Array) &&
+    !(value instanceof Uid);
+const text = (value) => typeof value === 'string' && value.trim() ? value.trim() : null;
+const date = (value) => (value instanceof Date ? value : null);
+const savedAt = (bookmark) => Date.parse(bookmark.addedAt);
+export class SafariBookmarksExtractor extends Extractor {
+    static source = 'safari';
+    static delivery = 'local';
+    static strategy = 'app-db';
+    static description = 'Bookmarks, favorites, and Reading List';
+    static default = true;
+    static recordTypes = ['bookmarks'];
+    // Bookmarks.plist holds what's saved now, in the folder it's in now. See
+    // Extractor.temporality.
+    static temporality = 'snapshot';
+    static schema = Extractor.schema.extend({
+        input: historyInput,
+        bookmarksFile: z
+            .string()
+            .optional()
+            .describe('Path to Safari Bookmarks.plist (default: the one beside History.db)'),
+        account: accountOption,
+    });
+    static defaultTransformer = SafariTransformer;
+    bookmarks = [];
+    changed;
+    /**
+     * Read the bookmarks file, by default the one in the folder of the
+     * History.db a run reads, so a copy of Safari's folder is read as one. A
+     * History.db without one beside it has no bookmarks to read.
+     */
+    async setup() {
+        await super.setup();
+        const { bookmarksFile, input } = this.config;
+        const path = bookmarksFile ?? join(dirname(input), 'Bookmarks.plist');
+        let bytes;
+        try {
+            bytes = await readFile(path);
+            this.changed = (await stat(path)).mtime;
+        }
+        catch (error) {
+            if (!bookmarksFile && error.code === 'ENOENT')
+                return;
+            throw fileError(error, path, 'bookmarks file');
+        }
+        this.bookmarks = this.read(parseBinaryPlist(bytes));
+    }
+    /**
+     * When the bookmarks last changed, not when the run started, so a re-read of
+     * unchanged bookmarks comes out the same.
+     */
+    asOfTime() {
+        return this.changed ?? super.asOfTime();
+    }
+    /** The bookmark's CloudKit record name, the same on each device, or its ID on this Mac. */
+    keyOf(record) {
+        return record.data.id;
+    }
+    occurredAt(record) {
+        return new Date(record.data.addedAt);
+    }
+    async *extract() {
+        const limit = this.getEffectiveLimit();
+        const account = this.config.account === undefined ? await getICloudAccount() : this.config.account;
+        for (const bookmark of this.inRange().slice(0, limit ?? undefined)) {
+            yield this.createRecord(bookmark, { source: 'safari', account });
+        }
+    }
+    /** The bookmarks a run reads and when, from the file setup() read. */
+    async survey(options = {}) {
+        const bookmarks = this.inRange();
+        const byMonth = {};
+        if (options.byMonth)
+            for (const { addedAt } of bookmarks)
+                countMonth(byMonth, new Date(addedAt));
+        return {
+            bookmarks: {
+                count: bookmarks.length,
+                ...(bookmarks.length > 0 && {
+                    earliest: new Date(bookmarks.at(-1).addedAt),
+                    latest: new Date(bookmarks[0].addedAt),
+                }),
+                ...(options.byMonth && { byMonth }),
+            },
+        };
+    }
+    /** The bookmarks within since/until, newest first. */
+    inRange() {
+        const { since, until } = this.config;
+        return this.bookmarks
+            .filter(b => (!since || savedAt(b) > since.getTime()) && (!until || savedAt(b) < until.getTime()))
+            .sort((a, b) => savedAt(b) - savedAt(a));
+    }
+    /**
+     * Every bookmark in the file that Safari dated: each Reading List item, and
+     * each bookmark Safari saved since it began dating them, but not those from
+     * before.
+     */
+    read(root) {
+        const bookmarks = [];
+        const walk = (node, folders) => {
+            for (const child of Array.isArray(node.Children) ? node.Children : []) {
+                if (!isNode(child))
+                    continue;
+                if (child.WebBookmarkType === 'WebBookmarkTypeList') {
+                    const title = text(child.Title) ?? '';
+                    walk(child, [...folders, { id: this.idOf(child), name: FOLDER_NAMES[title] ?? title }]);
+                }
+                else if (child.WebBookmarkType === 'WebBookmarkTypeLeaf' && text(child.URLString)) {
+                    const readingList = isNode(child.ReadingList) ? child.ReadingList : {};
+                    const addedAt = date(readingList.DateAdded) ?? date(child.dateAdded);
+                    if (!addedAt)
+                        continue;
+                    const uri = isNode(child.URIDictionary) ? child.URIDictionary : {};
+                    bookmarks.push({
+                        id: this.idOf(child),
+                        url: text(child.URLString),
+                        title: text(uri.title),
+                        addedAt: addedAt.toISOString(),
+                        folders,
+                    });
+                }
+            }
+        };
+        if (isNode(root))
+            walk(root, []);
+        return bookmarks;
+    }
+    idOf(node) {
+        return (isNode(node.Sync) && text(node.Sync.ServerID)) || String(node.WebBookmarkUUID);
+    }
+}

package/dist/connectors/SafariExtractor.d.ts

@@ -1,6 +1,29 @@
-import { Record } from '@chronicle.app/etl';
+import { Record, type Survey, type SurveyOptions } from '@chronicle.app/etl';
 import { SqliteExtractor } from '@chronicle.app/etl-sqlite';
+import { z } from 'zod';
 import SafariTransformer from './SafariTransformer.js';
+/** Safari's History.db; its Bookmarks.plist is in the same folder. */
+export declare const historyInput: z.ZodDefault<z.ZodString>;
+/**
+ * The Apple Account whose history and bookmarks these are. Omitted, it's the
+ * one signed in to this Mac; null, there's no one to name.
+ */
+export declare const accountOption: z.ZodOptional<z.ZodNullable<z.ZodObject<{
+    accountID: z.ZodString;
+    email: z.ZodString;
+    displayName: z.ZodString;
+    dsid: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    accountID: string;
+    email: string;
+    displayName: string;
+    dsid?: string | undefined;
+}, {
+    accountID: string;
+    email: string;
+    displayName: string;
+    dsid?: string | undefined;
+}>>>;
 export declare class SafariExtractor extends SqliteExtractor<typeof SafariExtractor> {
     static source: string;
     static delivery: "local";
@@ -9,13 +32,22 @@ export declare class SafariExtractor extends SqliteExtractor<typeof SafariExtrac
     static default: boolean;
     static recordTypes: string[];
     static newestFirst: boolean;
+    static databases: {
+        tables: {
+            history_items: string[];
+            history_visits: string[];
+        };
+    }[];
     static schema: any;
     static defaultTransformer: typeof SafariTransformer;
     /** history_visits.id — Safari's own visit id, AUTOINCREMENT so never reused. */
     keyOf(record: Record): string | null;
+    occurredAt(record: Record): Date | undefined;
     private visitFilter;
     extract(): AsyncGenerator<Record>;
     private processRow;
     determineCount(): Promise<number | null>;
+    /** The visits a run reads and when, with how many pages they were to. */
+    survey(options?: SurveyOptions): Promise<Survey | null>;
 }
 //# sourceMappingURL=SafariExtractor.d.ts.map

package/dist/connectors/SafariExtractor.js

@@ -1,9 +1,29 @@
-import { APPLE_EPOCH_OFFSET_SECONDS, SqliteExtractor, timeRangeConditions, unixToSafariTimestamp, iterateRows, } from '@chronicle.app/etl-sqlite';
-import { getICloudAccount } from '@chronicle.app/icloud';
+import { APPLE_EPOCH_OFFSET_SECONDS, SqliteExtractor, countsByMonth, monthSql, timeRangeConditions, unixToSafariTimestamp, iterateRows, } from '@chronicle.app/etl-sqlite';
+import { getICloudAccount } from '@chronicle.app/apple';
 import { z } from 'zod';
 import SafariTransformer from './SafariTransformer.js';
 // since/until bounds land on visit_time, which is seconds since the Apple epoch
 const safariBound = (date) => unixToSafariTimestamp(Math.floor(date.getTime() / 1000));
+// A visit's time as a record gives it: whole Unix seconds.
+const visitTime = `cast(history_visits.visit_time + ${APPLE_EPOCH_OFFSET_SECONDS} as integer)`;
+/** Safari's History.db; its Bookmarks.plist is in the same folder. */
+export const historyInput = z
+    .string()
+    .describe('Path to Safari History.db file')
+    .default(`${process.env.HOME}/Library/Safari/History.db`);
+/**
+ * The Apple Account whose history and bookmarks these are. Omitted, it's the
+ * one signed in to this Mac; null, there's no one to name.
+ */
+export const accountOption = z
+    .object({
+    accountID: z.string(),
+    email: z.string(),
+    displayName: z.string(),
+    dsid: z.string().optional(),
+})
+    .nullable()
+    .optional();
 export class SafariExtractor extends SqliteExtractor {
     static source = 'safari';
     static delivery = 'local';
@@ -14,20 +34,25 @@ export class SafariExtractor extends SqliteExtractor {
     // extract() orders by history_visits.visit_time desc, which with keyOf opts
     // this extractor into the frontier cursor (incremental catch-up import).
     static newestFirst = true;
+    // What extract() and survey() read, which a History.db must have.
+    static databases = [
+        {
+            tables: {
+                history_items: ['id', 'url'],
+                history_visits: [
+                    'id',
+                    'history_item',
+                    'visit_time',
+                    'title',
+                    'redirect_destination',
+                    'origin',
+                ],
+            },
+        },
+    ];
     static schema = SqliteExtractor.schema.omit({ input: true }).extend({
-        input: z
-            .string()
-            .describe('Path to Safari History.db file')
-            .default(`${process.env.HOME}/Library/Safari/History.db`),
-        account: z
-            .object({
-            accountID: z.string(),
-            email: z.string(),
-            displayName: z.string(),
-            dsid: z.string().optional(),
-        })
-            .nullable()
-            .optional(),
+        input: historyInput,
+        account: accountOption,
     });
     static defaultTransformer = SafariTransformer;
     /** history_visits.id — Safari's own visit id, AUTOINCREMENT so never reused. */
@@ -35,6 +60,9 @@ export class SafariExtractor extends SqliteExtractor {
         const visitId = record.data.visit_id;
         return visitId === null || visitId === undefined ? null : String(visitId);
     }
+    occurredAt(record) {
+        return new Date(record.data.unix_timestamp * 1000);
+    }
     // Visits that landed on a page (redirect hops excluded), within since/until.
     visitFilter() {
         const range = timeRangeConditions('history_visits.visit_time', this.config, {
@@ -56,7 +84,7 @@ export class SafariExtractor extends SqliteExtractor {
              history_visits.visit_time,
              history_visits.title,
              history_items.url,
-             cast(history_visits.visit_time + ${APPLE_EPOCH_OFFSET_SECONDS} as integer) AS unix_timestamp,
+             ${visitTime} AS unix_timestamp,
              datetime(history_visits.visit_time + ${APPLE_EPOCH_OFFSET_SECONDS}, 'unixepoch') AS visit_datetime,
              history_visits.origin
       FROM history_visits
@@ -107,4 +135,31 @@ export class SafariExtractor extends SqliteExtractor {
             return null;
         }
     }
+    /** The visits a run reads and when, with how many pages they were to. */
+    async survey(options = {}) {
+        if (!this.db)
+            return null;
+        const { where, values } = this.visitFilter();
+        const row = this.db
+            .prepare(`SELECT COUNT(*) AS n, MIN(${visitTime}) AS first, MAX(${visitTime}) AS last,
+          COUNT(DISTINCT history_items.id) AS pages FROM history_visits
+         LEFT JOIN history_items ON history_items.id = history_visits.history_item
+         WHERE ${where}`)
+            .get(...values);
+        const byMonth = options.byMonth &&
+            countsByMonth(this.db
+                .prepare(`SELECT ${monthSql(visitTime)} AS month, COUNT(*) AS n FROM history_visits
+             LEFT JOIN history_items ON history_items.id = history_visits.history_item
+             WHERE ${where} GROUP BY month`)
+                .all(...values));
+        return {
+            views: {
+                count: row.n,
+                ...(row.first !== null && { earliest: new Date(row.first * 1000) }),
+                ...(row.last !== null && { latest: new Date(row.last * 1000) }),
+                ...(byMonth && { byMonth }),
+                measures: [{ key: 'pages', label: 'pages', value: row.pages }],
+            },
+        };
+    }
 }

package/dist/connectors/SafariTransformer.d.ts

@@ -3,6 +3,7 @@ import { ActionAndChildren } from '@chronicle.app/schema';
 export default class SafariTransformer extends ChronicleTransformer {
     transform(record: Record): Promise<ActionAndChildren[]>;
     private buildViewActions;
+    private buildBookmarkAction;
     private buildWebPage;
 }
 //# sourceMappingURL=SafariTransformer.d.ts.map

package/dist/connectors/SafariTransformer.js

@@ -1,11 +1,14 @@
 import { ChronicleTransformer } from '@chronicle.app/etl';
-import { buildICloudPersonSchema } from '@chronicle.app/icloud';
+import { buildICloudPersonSchema } from '@chronicle.app/apple';
 export default class SafariTransformer extends ChronicleTransformer {
     async transform(record) {
         const actions = [];
         if (record.extraction.recordType === 'views') {
             actions.push(...(await this.buildViewActions(record)));
         }
+        else if (record.extraction.recordType === 'bookmarks') {
+            actions.push(await this.buildBookmarkAction(record));
+        }
         return actions;
     }
     async buildViewActions(record) {
@@ -25,6 +28,36 @@ export default class SafariTransformer extends ChronicleTransformer {
         actions.push(viewAction);
         return actions;
     }
+    // A bookmark, a favorite, and a Reading List item are each a page saved to
+    // a folder: the folder is the target, and a favorite's is Favorites.
+    async buildBookmarkAction(record) {
+        const bookmark = record.data;
+        const user = await buildICloudPersonSchema(record.context.account);
+        // Safari names its own folders the same in every account, such as
+        // `Favorites Bar`, so a folder's ID is unique only within its owner's.
+        let target;
+        for (const folder of bookmark.folders) {
+            target = {
+                '@type': 'Collection',
+                '@key': ['@type', 'source', 'inAccount[*].sourceId', 'sourceId'],
+                source: 'safari',
+                sourceId: folder.id,
+                name: folder.name,
+                ...(user && { inAccount: [user] }),
+                ...(target && { isPartOf: [target] }),
+            };
+        }
+        return {
+            '@type': 'BookmarkAction',
+            '@key': ['@type', 'source', 'sourceId'],
+            source: 'safari',
+            sourceId: bookmark.id,
+            timestamp: new Date(bookmark.addedAt),
+            ...(user && { agent: user }),
+            object: this.buildWebPage(record),
+            ...(target && { target }),
+        };
+    }
     buildWebPage(record) {
         return {
             '@type': 'Entity',

package/dist/index.d.ts

@@ -1,3 +1,4 @@
 export { SafariExtractor } from './connectors/SafariExtractor.js';
+export { SafariBookmarksExtractor } from './connectors/SafariBookmarksExtractor.js';
 export { default as SafariTransformer } from './connectors/SafariTransformer.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.js

@@ -1,3 +1,4 @@
 // Export extractors for plugin scanning
 export { SafariExtractor } from './connectors/SafariExtractor.js';
+export { SafariBookmarksExtractor } from './connectors/SafariBookmarksExtractor.js';
 export { default as SafariTransformer } from './connectors/SafariTransformer.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chronicle.app/safari",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "files": [
@@ -17,11 +17,11 @@
   "author": "Andrew Louis",
   "license": "MIT",
   "type": "module",
-  "description": "Chronicle Safari browsing history plugin",
+  "description": "Chronicle Safari browsing history, bookmarks, and Reading List plugin",
   "devDependencies": {
-    "@chronicle.app/etl": "0.4.0",
-    "@chronicle.app/schema": "0.4.0",
-    "@chronicle.app/tsconfig": "0.4.0",
+    "@chronicle.app/etl": "0.5.0",
+    "@chronicle.app/schema": "0.5.0",
+    "@chronicle.app/tsconfig": "0.5.0",
     "@types/node": "^22.10.1",
     "typescript": "^5.7.2"
   },
@@ -36,8 +36,8 @@
     "registry": "https://registry.npmjs.org/"
   },
   "dependencies": {
-    "@chronicle.app/etl-sqlite": "0.4.0",
-    "@chronicle.app/icloud": "0.4.0",
+    "@chronicle.app/etl-sqlite": "0.5.0",
+    "@chronicle.app/apple": "0.5.0",
     "zod": "^3.24.1"
   },
   "peerDependencies": {
@@ -61,9 +61,33 @@
           "app-db": {
             "delivery": "local",
             "recordTypes": [
-              "views"
+              "views",
+              "bookmarks"
             ],
-            "default": true
+            "default": true,
+            "locations": [
+              {
+                "path": "~/Library/Safari/History.db"
+              }
+            ],
+            "databases": [
+              {
+                "tables": {
+                  "history_items": [
+                    "id",
+                    "url"
+                  ],
+                  "history_visits": [
+                    "id",
+                    "history_item",
+                    "visit_time",
+                    "title",
+                    "redirect_destination",
+                    "origin"
+                  ]
+                }
+              }
+            ]
           }
         },
         "platforms": [