@chronicle.app/zotero 0.4.0 → 0.5.0

package/dist/connectors/ZoteroExtractor.d.ts

@@ -1,4 +1,4 @@
-import { Record } from '@chronicle.app/etl';
+import { Record, type Survey, type SurveyOptions } from '@chronicle.app/etl';
 import { SqliteExtractor } from '@chronicle.app/etl-sqlite';
 import ZoteroTransformer from './ZoteroTransformer.js';
 /**
@@ -69,5 +69,7 @@ export declare class ZoteroExtractor extends SqliteExtractor<typeof ZoteroExtrac
     private buildCollectionChain;
     private getItemCollections;
     determineCount(): Promise<number | null>;
+    /** The items a run reads of each record type, and when they were added. */
+    survey(options?: SurveyOptions): Promise<Survey | null>;
 }
 //# sourceMappingURL=ZoteroExtractor.d.ts.map

package/dist/connectors/ZoteroExtractor.js

@@ -1,16 +1,56 @@
-import { SystemInfo, assertReadable } from '@chronicle.app/etl';
-import { SqliteExtractor, timeRangeConditions, iterateRows } from '@chronicle.app/etl-sqlite';
+import { SystemInfo, assertReadable, } from '@chronicle.app/etl';
+import { SqliteExtractor, allRows, countsByMonth, monthSql, timeRangeConditions, iterateRows, } from '@chronicle.app/etl-sqlite';
 import { DatabaseSync } from 'node:sqlite';
 import { createHash, randomUUID } from 'node:crypto';
 import { constants as fsConstants, copyFileSync, readFileSync, rmSync, statSync } from 'node:fs';
-import { homedir, tmpdir } from 'node:os';
+import { tmpdir } from 'node:os';
 import { dirname, isAbsolute, join } from 'node:path';
 import { z } from 'zod';
-import ZoteroTransformer from './ZoteroTransformer.js';
+import ZoteroTransformer, { parseZoteroDate } from './ZoteroTransformer.js';
+import { zoteroDataDir } from '../dataDir.js';
 /** Zotero's UTC `YYYY-MM-DD HH:MM:SS` form, so string comparison against `dateModified` is correct. */
 function toSqliteTimestamp(d) {
     return d.toISOString().slice(0, 19).replace('T', ' ');
 }
+/**
+ * The items behind each record type, as `FROM … WHERE …` before since/until:
+ * the user library's, and none in the trash or under an item in it.
+ */
+const RECORD_ROWS = {
+    works: `
+      FROM items i
+      JOIN itemTypes it ON it.itemTypeID = i.itemTypeID
+      JOIN libraries l ON l.libraryID = i.libraryID
+      WHERE l.type = 'user'
+        AND i.itemID NOT IN (SELECT itemID FROM deletedItems)
+        AND it.typeName NOT IN ('attachment', 'annotation', 'note')`,
+    annotations: `
+      FROM itemAnnotations a
+      JOIN items i ON i.itemID = a.itemID
+      JOIN libraries l ON l.libraryID = i.libraryID
+      JOIN itemAttachments att ON att.itemID = a.parentItemID
+      JOIN items ai ON ai.itemID = att.itemID
+      WHERE l.type = 'user'
+        AND i.itemID NOT IN (SELECT itemID FROM deletedItems)
+        AND ai.itemID NOT IN (SELECT itemID FROM deletedItems)
+        AND (att.parentItemID IS NULL OR att.parentItemID NOT IN (SELECT itemID FROM deletedItems))`,
+    attachments: `
+      FROM itemAttachments att
+      JOIN items i ON i.itemID = att.itemID
+      JOIN libraries l ON l.libraryID = i.libraryID
+      WHERE l.type = 'user'
+        AND att.parentItemID IS NULL
+        AND i.itemID NOT IN (SELECT itemID FROM deletedItems)`,
+    notes: `
+      FROM items i
+      JOIN itemTypes it ON it.itemTypeID = i.itemTypeID
+      JOIN itemNotes n ON n.itemID = i.itemID
+      JOIN libraries l ON l.libraryID = i.libraryID
+      WHERE l.type = 'user'
+        AND it.typeName = 'note'
+        AND i.itemID NOT IN (SELECT itemID FROM deletedItems)
+        AND (n.parentItemID IS NULL OR n.parentItemID NOT IN (SELECT itemID FROM deletedItems))`,
+};
 function isSqliteBusy(err) {
     return (typeof err === 'object' &&
         err !== null &&
@@ -42,7 +82,7 @@ export class ZoteroExtractor extends SqliteExtractor {
         input: z
             .string()
             .describe('Path to the Zotero data directory (contains zotero.sqlite and storage/)')
-            .default(join(homedir(), 'Zotero')),
+            .default(() => zoteroDataDir()),
     });
     dataDir;
     localUserKey;
@@ -156,12 +196,7 @@ export class ZoteroExtractor extends SqliteExtractor {
         const sql = `
       SELECT i.itemID AS itemID, i.key AS key, i.dateAdded AS dateAdded, i.dateModified AS dateModified,
              it.typeName AS itemType
-      FROM items i
-      JOIN itemTypes it ON it.itemTypeID = i.itemTypeID
-      JOIN libraries l ON l.libraryID = i.libraryID
-      WHERE l.type = 'user'
-        AND i.itemID NOT IN (SELECT itemID FROM deletedItems)
-        AND it.typeName NOT IN ('attachment', 'annotation', 'note')${clause}
+      ${RECORD_ROWS.works}${clause}
       ORDER BY i.dateModified DESC`;
         for (const row of iterateRows(this.db.prepare(sql), ...params)) {
             if (this.shouldStopExtracting(this.extractedCount))
@@ -196,15 +231,7 @@ export class ZoteroExtractor extends SqliteExtractor {
              ai.itemID AS attItemID, ai.key AS attKey, ai.dateAdded AS attDateAdded,
              att.parentItemID AS workItemID, att.linkMode AS attLinkMode, att.contentType AS attContentType,
              att.path AS attPath, att.lastRead AS attLastRead
-      FROM itemAnnotations a
-      JOIN items i ON i.itemID = a.itemID
-      JOIN libraries l ON l.libraryID = i.libraryID
-      JOIN itemAttachments att ON att.itemID = a.parentItemID
-      JOIN items ai ON ai.itemID = att.itemID
-      WHERE l.type = 'user'
-        AND i.itemID NOT IN (SELECT itemID FROM deletedItems)
-        AND ai.itemID NOT IN (SELECT itemID FROM deletedItems)
-        AND (att.parentItemID IS NULL OR att.parentItemID NOT IN (SELECT itemID FROM deletedItems))${clause}
+      ${RECORD_ROWS.annotations}${clause}
       ORDER BY i.dateModified DESC`;
         for (const row of iterateRows(this.db.prepare(sql), ...params)) {
             if (this.shouldStopExtracting(this.extractedCount))
@@ -251,12 +278,7 @@ export class ZoteroExtractor extends SqliteExtractor {
       SELECT i.itemID AS itemID, i.key AS key, i.dateAdded AS dateAdded, i.dateModified AS dateModified,
              att.parentItemID AS parentItemID, att.linkMode AS linkMode, att.contentType AS contentType,
              att.path AS path, att.lastRead AS lastRead
-      FROM itemAttachments att
-      JOIN items i ON i.itemID = att.itemID
-      JOIN libraries l ON l.libraryID = i.libraryID
-      WHERE l.type = 'user'
-        AND att.parentItemID IS NULL
-        AND i.itemID NOT IN (SELECT itemID FROM deletedItems)${clause}
+      ${RECORD_ROWS.attachments}${clause}
       ORDER BY i.dateModified DESC`;
         for (const row of iterateRows(this.db.prepare(sql), ...params)) {
             if (this.shouldStopExtracting(this.extractedCount))
@@ -281,14 +303,7 @@ export class ZoteroExtractor extends SqliteExtractor {
         const sql = `
       SELECT i.itemID AS itemID, i.key AS key, i.dateAdded AS dateAdded, i.dateModified AS dateModified,
              n.parentItemID AS parentItemID, n.note AS note, n.title AS title
-      FROM items i
-      JOIN itemTypes it ON it.itemTypeID = i.itemTypeID
-      JOIN itemNotes n ON n.itemID = i.itemID
-      JOIN libraries l ON l.libraryID = i.libraryID
-      WHERE l.type = 'user'
-        AND it.typeName = 'note'
-        AND i.itemID NOT IN (SELECT itemID FROM deletedItems)
-        AND (n.parentItemID IS NULL OR n.parentItemID NOT IN (SELECT itemID FROM deletedItems))${clause}
+      ${RECORD_ROWS.notes}${clause}
       ORDER BY i.dateModified DESC`;
         for (const row of iterateRows(this.db.prepare(sql), ...params)) {
             if (this.shouldStopExtracting(this.extractedCount))
@@ -498,55 +513,39 @@ export class ZoteroExtractor extends SqliteExtractor {
         if (!this.db)
             return null;
         try {
-            const worksFilter = this.sinceUntilFilter('i.dateModified');
-            const works = this.db
-                .prepare(`SELECT COUNT(*) AS n
-           FROM items i
-           JOIN itemTypes it ON it.itemTypeID = i.itemTypeID
-           JOIN libraries l ON l.libraryID = i.libraryID
-           WHERE l.type = 'user'
-             AND i.itemID NOT IN (SELECT itemID FROM deletedItems)
-             AND it.typeName NOT IN ('attachment', 'annotation', 'note')${worksFilter.clause}`)
-                .get(...worksFilter.params);
-            const annotationsFilter = this.sinceUntilFilter('i.dateModified');
-            const annotations = this.db
-                .prepare(`SELECT COUNT(*) AS n
-           FROM itemAnnotations a
-           JOIN items i ON i.itemID = a.itemID
-           JOIN libraries l ON l.libraryID = i.libraryID
-           JOIN itemAttachments att ON att.itemID = a.parentItemID
-           JOIN items ai ON ai.itemID = att.itemID
-           WHERE l.type = 'user'
-             AND i.itemID NOT IN (SELECT itemID FROM deletedItems)
-             AND ai.itemID NOT IN (SELECT itemID FROM deletedItems)
-             AND (att.parentItemID IS NULL OR att.parentItemID NOT IN (SELECT itemID FROM deletedItems))${annotationsFilter.clause}`)
-                .get(...annotationsFilter.params);
-            const attachmentsFilter = this.sinceUntilFilter('i.dateModified');
-            const attachments = this.db
-                .prepare(`SELECT COUNT(*) AS n
-           FROM itemAttachments att
-           JOIN items i ON i.itemID = att.itemID
-           JOIN libraries l ON l.libraryID = i.libraryID
-           WHERE l.type = 'user'
-             AND att.parentItemID IS NULL
-             AND i.itemID NOT IN (SELECT itemID FROM deletedItems)${attachmentsFilter.clause}`)
-                .get(...attachmentsFilter.params);
-            const notesFilter = this.sinceUntilFilter('i.dateModified');
-            const notes = this.db
-                .prepare(`SELECT COUNT(*) AS n
-           FROM items i
-           JOIN itemTypes it ON it.itemTypeID = i.itemTypeID
-           JOIN itemNotes n ON n.itemID = i.itemID
-           JOIN libraries l ON l.libraryID = i.libraryID
-           WHERE l.type = 'user'
-             AND it.typeName = 'note'
-             AND i.itemID NOT IN (SELECT itemID FROM deletedItems)
-             AND (n.parentItemID IS NULL OR n.parentItemID NOT IN (SELECT itemID FROM deletedItems))${notesFilter.clause}`)
-                .get(...notesFilter.params);
-            return works.n + annotations.n + attachments.n + notes.n;
+            const { clause, params } = this.sinceUntilFilter('i.dateModified');
+            let total = 0;
+            for (const rows of Object.values(RECORD_ROWS)) {
+                const { n } = this.db.prepare(`SELECT COUNT(*) AS n ${rows}${clause}`).get(...params);
+                total += n;
+            }
+            return total;
         }
         catch {
             return null;
         }
     }
+    /** The items a run reads of each record type, and when they were added. */
+    async survey(options = {}) {
+        if (!this.db)
+            return null;
+        const { clause, params } = this.sinceUntilFilter('i.dateModified');
+        // dateAdded is UTC text, so its month is from its Unix seconds.
+        const month = monthSql("CAST(strftime('%s', i.dateAdded) AS INTEGER)");
+        const survey = {};
+        for (const [recordType, rows] of Object.entries(RECORD_ROWS)) {
+            const { n, first, last } = this.db
+                .prepare(`SELECT COUNT(*) AS n, MIN(i.dateAdded) AS first, MAX(i.dateAdded) AS last ${rows}${clause}`)
+                .get(...params);
+            const byMonth = options.byMonth &&
+                countsByMonth(allRows(this.db.prepare(`SELECT ${month} AS month, COUNT(*) AS n ${rows}${clause} GROUP BY month`), ...params));
+            survey[recordType] = {
+                count: n,
+                ...(first !== null && { earliest: parseZoteroDate(first) }),
+                ...(last !== null && { latest: parseZoteroDate(last) }),
+                ...(byMonth && { byMonth }),
+            };
+        }
+        return survey;
+    }
 }

package/dist/connectors/ZoteroLocator.d.ts

@@ -0,0 +1,12 @@
+import { Locator, type Candidate, type LocationMatch } from '@chronicle.app/etl';
+/**
+ * Zotero's library where its preferences say it is, which is where a run
+ * reads it, and one left in the default `~/Zotero` the manifest names, if
+ * that's another.
+ */
+export declare class ZoteroLocator extends Locator {
+    static source: string;
+    static strategy: string;
+    locate(matches: LocationMatch[]): AsyncGenerator<Candidate>;
+}
+//# sourceMappingURL=ZoteroLocator.d.ts.map

package/dist/connectors/ZoteroLocator.js

@@ -0,0 +1,20 @@
+import { dirname, join } from 'node:path';
+import { Locator } from '@chronicle.app/etl';
+import { zoteroDataDir } from '../dataDir.js';
+/**
+ * Zotero's library where its preferences say it is, which is where a run
+ * reads it, and one left in the default `~/Zotero` the manifest names, if
+ * that's another.
+ */
+export class ZoteroLocator extends Locator {
+    static source = 'zotero';
+    static strategy = 'app-db';
+    async *locate(matches) {
+        const libraries = new Set([
+            join(zoteroDataDir(this.context.home), 'zotero.sqlite'),
+            ...matches.map(match => match.path),
+        ]);
+        for (const file of libraries)
+            yield { config: { input: dirname(file) }, path: file };
+    }
+}

package/dist/connectors/ZoteroTransformer.d.ts

@@ -1,5 +1,7 @@
 import { ChronicleTransformer, Record } from '@chronicle.app/etl';
 import { ActionAndChildren } from '@chronicle.app/schema';
+/** Zotero timestamps are UTC `YYYY-MM-DD HH:MM:SS`, no zone marker. */
+export declare function parseZoteroDate(raw: string): Date;
 /**
  * A local Zotero library (`zotero.sqlite`) → Chronicle JSON-LD.
  *

package/dist/connectors/ZoteroTransformer.js

@@ -2,7 +2,7 @@ import path from 'node:path';
 import { ChronicleTransformer } from '@chronicle.app/etl';
 const SOURCE = 'zotero';
 /** Zotero timestamps are UTC `YYYY-MM-DD HH:MM:SS`, no zone marker. */
-function parseZoteroDate(raw) {
+export function parseZoteroDate(raw) {
     return new Date(`${raw.replace(' ', 'T')}Z`);
 }
 /** Zotero itemType → the Chronicle work subtype (every value here exists in the schema). */
@@ -239,9 +239,9 @@ export default class ZoteroTransformer extends ChronicleTransformer {
         const actions = [
             {
                 '@type': 'BookmarkAction',
-                '@key': ['@type', 'source', 'sourceId', 'object.inRealm.handle'],
+                // Keyed by the work it saved.
+                '@key': ['@type', 'source', 'object.sourceId', 'object.inRealm.handle'],
                 source: SOURCE,
-                sourceId: work.key,
                 timestamp: parseZoteroDate(work.dateAdded),
                 agent: me,
                 object: workEntity,
@@ -255,7 +255,7 @@ export default class ZoteroTransformer extends ChronicleTransformer {
                 // so each viewing independently witnesses the merge regardless of
                 // fold order.
                 const facet = this.buildFileFacet(attachment, context, workRef);
-                actions.push(this.buildViewAction(attachment.key, attachment.lastRead, facet, me));
+                actions.push(this.buildViewAction(attachment.lastRead, facet, me));
             }
         }
         return actions;
@@ -270,12 +270,12 @@ export default class ZoteroTransformer extends ChronicleTransformer {
      * "currently reading" state. A single sighted open is one-shot
      * consumption; `ViewAction` is its honest weight.
      */
-    buildViewAction(sourceId, lastRead, object, agent) {
+    buildViewAction(lastRead, object, agent) {
         return {
             '@type': 'ViewAction',
-            '@key': ['@type', 'source', 'sourceId', 'object.inRealm.handle', 'timestamp'],
+            // Keyed by the file it opened, and when.
+            '@key': ['@type', 'source', 'object.sourceId', 'object.inRealm.handle', 'timestamp'],
             source: SOURCE,
-            sourceId,
             timestamp: new Date(lastRead * 1000),
             agent,
             object,
@@ -754,29 +754,37 @@ export default class ZoteroTransformer extends ChronicleTransformer {
                 quotation.selector = [selector];
             actions.push({
                 '@type': 'QuoteAction',
-                '@key': ['@type', 'source', 'sourceId', 'result.inRealm.handle'],
+                // Keyed by the quotation it made, which has the annotation's key.
+                '@key': ['@type', 'source', 'result.sourceId', 'result.inRealm.handle'],
                 source: SOURCE,
-                sourceId: annotation.key,
                 timestamp: parseZoteroDate(annotation.dateAdded),
                 agent: me,
                 result: quotation,
             });
         }
         if (hasComment) {
+            // A comment on a highlight is keyed by the quotation it's about, which
+            // has the annotation's key. A comment on its own has the key itself.
             const comment = {
                 '@type': 'Comment',
-                '@key': ['@type', 'source', 'sourceId', 'inRealm.handle'],
+                ...(quotation
+                    ? { '@key': ['@type', 'source', 'about[*].sourceId', 'inRealm.handle'] }
+                    : {
+                        '@key': ['@type', 'source', 'sourceId', 'inRealm.handle'],
+                        sourceId: annotation.key,
+                    }),
                 source: SOURCE,
-                sourceId: annotation.key,
                 inRealm: realm,
                 description: annotation.comment,
                 about: [quotation ?? subjectRef],
             };
             actions.push({
                 '@type': 'AnnotateAction',
-                '@key': ['@type', 'source', 'sourceId', 'result.inRealm.handle'],
+                // Keyed by the comment it made.
+                '@key': quotation
+                    ? ['@type', 'source', 'result.about[*].sourceId', 'result.inRealm.handle']
+                    : ['@type', 'source', 'result.sourceId', 'result.inRealm.handle'],
                 source: SOURCE,
-                sourceId: annotation.key,
                 timestamp: parseZoteroDate(annotation.dateAdded),
                 agent: me,
                 result: comment,
@@ -831,16 +839,16 @@ export default class ZoteroTransformer extends ChronicleTransformer {
         const actions = [
             {
                 '@type': 'BookmarkAction',
-                '@key': ['@type', 'source', 'sourceId', 'object.inRealm.handle'],
+                // Keyed by the file it saved.
+                '@key': ['@type', 'source', 'object.sourceId', 'object.inRealm.handle'],
                 source: SOURCE,
-                sourceId: attachment.key,
                 timestamp: parseZoteroDate(attachment.dateAdded),
                 agent: me,
                 object: entity,
             },
         ];
         if (attachment.lastRead) {
-            actions.push(this.buildViewAction(attachment.key, attachment.lastRead, this.buildAttachmentRef(attachment, context), me));
+            actions.push(this.buildViewAction(attachment.lastRead, this.buildAttachmentRef(attachment, context), me));
         }
         return actions;
     }
@@ -929,9 +937,9 @@ export default class ZoteroTransformer extends ChronicleTransformer {
         return [
             {
                 '@type': 'AnnotateAction',
-                '@key': ['@type', 'source', 'sourceId', 'result.inRealm.handle'],
+                // Keyed by the note it made.
+                '@key': ['@type', 'source', 'result.sourceId', 'result.inRealm.handle'],
                 source: SOURCE,
-                sourceId: note.key,
                 timestamp: parseZoteroDate(note.dateAdded),
                 agent: me,
                 result: comment,

package/dist/dataDir.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Zotero's data folder: the one a profile's preferences name, when it was
+ * moved from the default, else `~/Zotero`. Read the way Zotero reads it, so
+ * a run and a scan find the same library.
+ */
+export declare function zoteroDataDir(home?: string): string;
+//# sourceMappingURL=dataDir.d.ts.map

package/dist/dataDir.js

@@ -0,0 +1,51 @@
+import { readFileSync, readdirSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+/** Where Zotero keeps its profiles, on macOS and on Linux. */
+const PROFILES = [
+    ['Library', 'Application Support', 'Zotero', 'Profiles'],
+    ['.zotero', 'zotero'],
+];
+/** A preference's value from a profile's `prefs.js`, as the JSON Zotero writes it. */
+function pref(prefs, name) {
+    const value = prefs.match(new RegExp(`user_pref\\("${name.replaceAll('.', '\\.')}",\\s*(.+?)\\);`));
+    if (!value)
+        return undefined;
+    try {
+        return JSON.parse(value[1]);
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * Zotero's data folder: the one a profile's preferences name, when it was
+ * moved from the default, else `~/Zotero`. Read the way Zotero reads it, so
+ * a run and a scan find the same library.
+ */
+export function zoteroDataDir(home = homedir()) {
+    for (const parts of PROFILES) {
+        const root = join(home, ...parts);
+        let profiles;
+        try {
+            profiles = readdirSync(root).sort();
+        }
+        catch {
+            continue;
+        }
+        for (const profile of profiles) {
+            let prefs;
+            try {
+                prefs = readFileSync(join(root, profile, 'prefs.js'), 'utf8');
+            }
+            catch {
+                continue;
+            }
+            const dir = pref(prefs, 'extensions.zotero.dataDir');
+            if (pref(prefs, 'extensions.zotero.useDataDir') === true && typeof dir === 'string') {
+                return dir;
+            }
+        }
+    }
+    return join(home, 'Zotero');
+}

package/dist/index.d.ts

@@ -1,3 +1,5 @@
 export { ZoteroExtractor } from './connectors/ZoteroExtractor.js';
+export { ZoteroLocator } from './connectors/ZoteroLocator.js';
+export { zoteroDataDir } from './dataDir.js';
 export { default as ZoteroTransformer } from './connectors/ZoteroTransformer.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.js

@@ -1,2 +1,4 @@
 export { ZoteroExtractor } from './connectors/ZoteroExtractor.js';
+export { ZoteroLocator } from './connectors/ZoteroLocator.js';
+export { zoteroDataDir } from './dataDir.js';
 export { default as ZoteroTransformer } from './connectors/ZoteroTransformer.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chronicle.app/zotero",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "files": [
@@ -22,9 +22,9 @@
   "type": "module",
   "description": "Chronicle plugin for a local Zotero library: works, highlights, notes, and reading events",
   "devDependencies": {
-    "@chronicle.app/etl": "0.4.0",
-    "@chronicle.app/schema": "0.4.0",
-    "@chronicle.app/tsconfig": "0.4.0",
+    "@chronicle.app/etl": "0.5.0",
+    "@chronicle.app/schema": "0.5.0",
+    "@chronicle.app/tsconfig": "0.5.0",
     "@types/node": "^22.10.1",
     "typescript": "^5.7.2"
   },
@@ -39,7 +39,7 @@
     "registry": "https://registry.npmjs.org/"
   },
   "dependencies": {
-    "@chronicle.app/etl-sqlite": "0.4.0",
+    "@chronicle.app/etl-sqlite": "0.5.0",
     "zod": "^3.24.1"
   },
   "peerDependencies": {
@@ -68,7 +68,12 @@
               "attachments",
               "notes"
             ],
-            "default": true
+            "default": true,
+            "locations": [
+              {
+                "path": "~/Zotero/zotero.sqlite"
+              }
+            ]
           }
         }
       }