npm - @openneuro/server - Versions diffs - 4.46.0 → 4.47.0-alpha.0 - Mend

@openneuro/server 4.46.0 → 4.47.0-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (32) hide show

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "@openneuro/server",
-  "version": "4.46.0",
+  "version": "4.47.0-alpha.0",
   "description": "Core service for the OpenNeuro platform.",
   "license": "MIT",
   "main": "src/server.js",
@@ -21,7 +21,7 @@
     "@elastic/elasticsearch": "8.13.1",
     "@graphql-tools/schema": "^10.0.0",
     "@keyv/redis": "^4.5.0",
-    "@openneuro/search": "^4.46.0",
+    "@openneuro/search": "^4.47.0-alpha.0",
     "@sentry/node": "^10.37.0",
     "@sentry/profiling-node": "^10.37.0",
     "base64url": "^3.0.0",
@@ -36,7 +36,7 @@
     "graphql-compose": "9.0.10",
     "graphql-iso-date": "^3.6.1",
     "graphql-tools": "9.0.0",
-    "immutable": "^3.8.2",
+    "immutable": "^4.3.8",
     "ioredis": "^5.6.1",
     "js-yaml": "^4.1.0",
     "jsdom": "24.0.0",
@@ -89,5 +89,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "b40f6f5b597cbb722d09497182059a5157abf110"
+  "gitHead": "0cce162b7c6dd6f62e16b2c50e68c77fb1048a7e"
 }

package/src/config.ts CHANGED Viewed

@@ -1,4 +1,5 @@
 const config = {
+  // Base URL for the site (e.g. `https://openneuro.org`)
   url: process.env.CRN_SERVER_URL,
   port: 8111,
   apiPrefix: "/crn/",
@@ -8,6 +9,7 @@ const config = {
     "Access-Control-Allow-Methods": "GET, POST, OPTIONS, PUT, PATCH, DELETE",
     "Access-Control-Allow-Headers": "content-type, Authorization",
   },
+  // Deprecated analysis feature flag
   analysis: {
     enabled: process.env.ANALYSIS_ENABLED,
   },

package/src/datalad/__tests__/contributors.spec.ts CHANGED Viewed

@@ -6,6 +6,7 @@ import { fileUrl } from "../files"
 import { datasetOrSnapshot } from "../../utils/datasetOrSnapshot"
 import { contributors } from "../contributors"
+vi.mock("../../config.ts")
 vi.mock("../../libs/authentication/jwt", () => ({
   sign: vi.fn(() => "mock_jwt_token"),
   verify: vi.fn(() => ({ userId: "mock_user_id" })),

package/src/datalad/__tests__/dataRetentionNotifications.spec.ts ADDED Viewed

@@ -0,0 +1,315 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { vi } from "vitest"
+vi.mock("ioredis")
+vi.mock("../../config.ts")
+vi.mock("../../libs/notifications.ts", () => ({
+  default: { send: vi.fn() },
+}))
+vi.mock("../../libs/email/templates/draft-retention-warning", () => ({
+  draftRetentionWarning: vi.fn(() => "<html>warning</html>"),
+}))
+vi.mock("../../libs/email/templates/draft-retention-deletion", () => ({
+  draftRetentionDeletion: vi.fn(() => "<html>deletion</html>"),
+}))
+vi.mock("../../libs/email/templates/snapshot-reminder", () => ({
+  snapshotReminder: vi.fn(() => "<html>snapshot</html>"),
+}))
+import { MongoMemoryServer } from "mongodb-memory-server"
+import { connect, disconnect } from "mongoose"
+import notifications from "../../libs/notifications"
+import DataRetention from "../../models/dataRetention"
+import Permission from "../../models/permission"
+import User from "../../models/user"
+import { checkDataRetentionNotifications } from "../dataRetentionNotifications"
+import * as draftModule from "../draft"
+import * as snapshotsModule from "../snapshots"
+const DAY = 24 * 60 * 60 * 1000
+const TEST_DATASET = "ds000001"
+const TEST_HEXSHA = "abc123"
+const TEST_USER = {
+  id: "user1",
+  email: "test@example.com",
+  name: "Test User",
+}
+function daysAgo(days: number): Date {
+  return new Date(Date.now() - days * DAY)
+}
+describe("checkDataRetentionNotifications", () => {
+  let mongod: MongoMemoryServer
+  beforeAll(async () => {
+    mongod = await MongoMemoryServer.create()
+    await connect(mongod.getUri())
+  })
+  afterAll(async () => {
+    await disconnect()
+    await mongod.stop()
+  })
+  beforeEach(async () => {
+    await DataRetention.deleteMany({})
+    await Permission.deleteMany({})
+    await User.deleteMany({})
+    vi.mocked(notifications.send).mockClear()
+    // Seed a user with write permission
+    await User.create(TEST_USER)
+    await Permission.create({
+      datasetId: TEST_DATASET,
+      userId: TEST_USER.id,
+      level: "rw",
+    })
+  })
+  function mockDraft(modified: Date, hexsha = TEST_HEXSHA) {
+    vi.spyOn(draftModule, "getDraftInfo").mockResolvedValue({
+      modified,
+      hexsha,
+    } as any)
+  }
+  function mockSnapshots(snapshots: { hexsha: string }[] = []) {
+    vi.spyOn(snapshotsModule, "getSnapshots").mockResolvedValue(
+      snapshots as any,
+    )
+  }
+  it("does nothing when draft matches the latest snapshot", async () => {
+    mockDraft(daysAgo(30), TEST_HEXSHA)
+    mockSnapshots([{ hexsha: TEST_HEXSHA }])
+    await checkDataRetentionNotifications(TEST_DATASET)
+    expect(notifications.send).not.toHaveBeenCalled()
+  })
+  it("does nothing when draft is less than 1 day old", async () => {
+    mockDraft(new Date()) // just now
+    mockSnapshots([])
+    await checkDataRetentionNotifications(TEST_DATASET)
+    expect(notifications.send).not.toHaveBeenCalled()
+  })
+  it("sends snapshot reminder after 1 day with no snapshot", async () => {
+    mockDraft(daysAgo(2))
+    mockSnapshots([])
+    await checkDataRetentionNotifications(TEST_DATASET)
+    expect(notifications.send).toHaveBeenCalledTimes(1)
+    expect(notifications.send).toHaveBeenCalledWith(
+      expect.objectContaining({
+        _id: expect.stringContaining("no_snapshot_reminder"),
+      }),
+    )
+  })
+  it("skips snapshot reminder when draft is already 14+ days old", async () => {
+    mockDraft(daysAgo(15))
+    mockSnapshots([])
+    await checkDataRetentionNotifications(TEST_DATASET)
+    // Should send only the 14-day retention warning, not the snapshot reminder
+    expect(notifications.send).toHaveBeenCalledTimes(1)
+    expect(notifications.send).toHaveBeenCalledWith(
+      expect.objectContaining({
+        _id: expect.stringContaining("retention_14day"),
+      }),
+    )
+  })
+  it("sends 14-day warning when draft is 14+ days old", async () => {
+    mockDraft(daysAgo(15))
+    mockSnapshots([{ hexsha: "other" }])
+    await checkDataRetentionNotifications(TEST_DATASET)
+    expect(notifications.send).toHaveBeenCalledTimes(1)
+    expect(notifications.send).toHaveBeenCalledWith(
+      expect.objectContaining({
+        _id: expect.stringContaining("retention_14day"),
+      }),
+    )
+  })
+  it("sends notices in order even when draft is already past 28 days", async () => {
+    mockDraft(daysAgo(35))
+    mockSnapshots([{ hexsha: "other" }])
+    // First call: should send 14-day warning, NOT deletion
+    await checkDataRetentionNotifications(TEST_DATASET)
+    expect(notifications.send).toHaveBeenCalledTimes(1)
+    expect(notifications.send).toHaveBeenCalledWith(
+      expect.objectContaining({
+        _id: expect.stringContaining("retention_14day"),
+      }),
+    )
+    const record = await DataRetention.findOne({ datasetId: TEST_DATASET })
+      .exec()
+    expect(record.notifiedAt14Days).toBeTruthy()
+    expect(record.notifiedAt7Days).toBeNull()
+    expect(record.notifiedAtDeletion).toBeNull()
+  })
+  it("does not send 7-day warning until 7 days after 14-day notice", async () => {
+    mockDraft(daysAgo(35))
+    mockSnapshots([{ hexsha: "other" }])
+    // Seed a 14-day notice sent only 3 days ago
+    await DataRetention.create({
+      datasetId: TEST_DATASET,
+      hexsha: TEST_HEXSHA,
+      notifiedAt14Days: daysAgo(3),
+    })
+    await checkDataRetentionNotifications(TEST_DATASET)
+    // No new notice should be sent (14-day already sent, 7-day not due yet)
+    expect(notifications.send).not.toHaveBeenCalled()
+  })
+  it("sends 7-day warning 7 days after 14-day notice", async () => {
+    mockDraft(daysAgo(35))
+    mockSnapshots([{ hexsha: "other" }])
+    // Seed a 14-day notice sent 8 days ago
+    await DataRetention.create({
+      datasetId: TEST_DATASET,
+      hexsha: TEST_HEXSHA,
+      notifiedAt14Days: daysAgo(8),
+    })
+    await checkDataRetentionNotifications(TEST_DATASET)
+    expect(notifications.send).toHaveBeenCalledTimes(1)
+    expect(notifications.send).toHaveBeenCalledWith(
+      expect.objectContaining({
+        _id: expect.stringContaining("retention_7day"),
+      }),
+    )
+  })
+  it("sends deletion notice 7 days after 7-day warning", async () => {
+    mockDraft(daysAgo(40))
+    mockSnapshots([{ hexsha: "other" }])
+    // Seed both prior notices
+    await DataRetention.create({
+      datasetId: TEST_DATASET,
+      hexsha: TEST_HEXSHA,
+      notifiedAt14Days: daysAgo(15),
+      notifiedAt7Days: daysAgo(8),
+    })
+    await checkDataRetentionNotifications(TEST_DATASET)
+    expect(notifications.send).toHaveBeenCalledTimes(1)
+    expect(notifications.send).toHaveBeenCalledWith(
+      expect.objectContaining({
+        _id: expect.stringContaining("retention_deletion"),
+      }),
+    )
+  })
+  it("does not send deletion notice until 7 days after 7-day warning", async () => {
+    mockDraft(daysAgo(40))
+    mockSnapshots([{ hexsha: "other" }])
+    await DataRetention.create({
+      datasetId: TEST_DATASET,
+      hexsha: TEST_HEXSHA,
+      notifiedAt14Days: daysAgo(10),
+      notifiedAt7Days: daysAgo(3),
+    })
+    await checkDataRetentionNotifications(TEST_DATASET)
+    expect(notifications.send).not.toHaveBeenCalled()
+  })
+  it("resets retention notices when draft hexsha changes", async () => {
+    mockDraft(daysAgo(15), "new_hexsha")
+    mockSnapshots([{ hexsha: "other" }])
+    // Pre-existing record with old hexsha and all notices sent
+    await DataRetention.create({
+      datasetId: TEST_DATASET,
+      hexsha: "old_hexsha",
+      notifiedAt14Days: daysAgo(10),
+      notifiedAt7Days: daysAgo(3),
+      notifiedAtDeletion: daysAgo(1),
+    })
+    await checkDataRetentionNotifications(TEST_DATASET)
+    // Should send 14-day notice again (reset due to new hexsha)
+    expect(notifications.send).toHaveBeenCalledTimes(1)
+    expect(notifications.send).toHaveBeenCalledWith(
+      expect.objectContaining({
+        _id: expect.stringContaining("retention_14day"),
+      }),
+    )
+    const record = await DataRetention.findOne({ datasetId: TEST_DATASET })
+      .exec()
+    expect(record.hexsha).toBe("new_hexsha")
+    expect(record.notifiedAt14Days).toBeTruthy()
+    expect(record.notifiedAt7Days).toBeNull()
+    expect(record.notifiedAtDeletion).toBeNull()
+  })
+  it("walks through the full notification sequence with real delays", async () => {
+    mockSnapshots([{ hexsha: "other" }])
+    // Day 14: first warning
+    mockDraft(daysAgo(15))
+    await checkDataRetentionNotifications(TEST_DATASET)
+    expect(notifications.send).toHaveBeenCalledTimes(1)
+    expect(notifications.send).toHaveBeenLastCalledWith(
+      expect.objectContaining({
+        _id: expect.stringContaining("retention_14day"),
+      }),
+    )
+    // Simulate 7 days passing by backdating the 14-day notice
+    await DataRetention.updateOne(
+      { datasetId: TEST_DATASET },
+      { notifiedAt14Days: daysAgo(8) },
+    ).exec()
+    vi.mocked(notifications.send).mockClear()
+    // Day 21: 7-day warning
+    mockDraft(daysAgo(22))
+    await checkDataRetentionNotifications(TEST_DATASET)
+    expect(notifications.send).toHaveBeenCalledTimes(1)
+    expect(notifications.send).toHaveBeenLastCalledWith(
+      expect.objectContaining({
+        _id: expect.stringContaining("retention_7day"),
+      }),
+    )
+    // Simulate 7 more days passing
+    await DataRetention.updateOne(
+      { datasetId: TEST_DATASET },
+      { notifiedAt7Days: daysAgo(8) },
+    ).exec()
+    vi.mocked(notifications.send).mockClear()
+    // Day 28: deletion notice
+    mockDraft(daysAgo(30))
+    await checkDataRetentionNotifications(TEST_DATASET)
+    expect(notifications.send).toHaveBeenCalledTimes(1)
+    expect(notifications.send).toHaveBeenLastCalledWith(
+      expect.objectContaining({
+        _id: expect.stringContaining("retention_deletion"),
+      }),
+    )
+    // No further notices after deletion
+    vi.mocked(notifications.send).mockClear()
+    await checkDataRetentionNotifications(TEST_DATASET)
+    expect(notifications.send).not.toHaveBeenCalled()
+  })
+})

package/src/datalad/__tests__/dataset.spec.ts CHANGED Viewed

@@ -1,3 +1,4 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
 import { vi } from "vitest"
 import request from "superagent"
 import { createDataset, datasetsFilter, testBlacklist } from "../dataset"
@@ -11,6 +12,8 @@ vi.mock("ioredis")
 vi.mock("../../libs/redis")
 vi.mock("../../config.ts")
 vi.mock("../../libs/notifications")
+vi.mock("../draft")
+vi.mock("../snapshots")
 describe("dataset model operations", () => {
   describe("createDataset()", () => {

package/src/datalad/dataRetentionNotifications.ts ADDED Viewed

@@ -0,0 +1,160 @@
+import config from "../config"
+import notifications from "../libs/notifications"
+import User from "../models/user"
+import Permission from "../models/permission"
+import DataRetention from "../models/dataRetention"
+import { getDraftInfo } from "./draft"
+import { getSnapshots } from "./snapshots"
+import { draftRetentionWarning } from "../libs/email/templates/draft-retention-warning"
+import { draftRetentionDeletion } from "../libs/email/templates/draft-retention-deletion"
+import { snapshotReminder } from "../libs/email/templates/snapshot-reminder"
+const DAY = 24 * 60 * 60 * 1000
+/**
+ * Notify all users with write or admin access to a dataset.
+ */
+async function notifyWriteUsers(
+  datasetId: string,
+  makeEmail: (user: { _id: string; email: string; name: string }) => object,
+) {
+  const permissions = await Permission.find({
+    datasetId,
+    level: { $in: ["rw", "admin"] },
+  }).exec()
+  for (const permission of permissions) {
+    const user = await User.findOne({ id: permission.userId }).exec()
+    if (user) {
+      notifications.send(makeEmail(user))
+    }
+  }
+}
+/**
+ * Check and send data retention notifications for a dataset.
+ *
+ * Retention warnings (14-day, 7-day, deletion) are tied to the current draft
+ * hexsha and reset whenever the draft changes. The no-snapshot 24h notice is
+ * sent once regardless of future draft changes.
+ */
+export async function checkDataRetentionNotifications(
+  datasetId: string,
+): Promise<void> {
+  const draft = await getDraftInfo(datasetId)
+  const snapshots = await getSnapshots(datasetId)
+  const lastSnapshot = snapshots?.length
+    ? snapshots[snapshots.length - 1]
+    : null
+  // Draft is in sync with the last snapshot — no retention action needed
+  if (lastSnapshot && draft.hexsha === lastSnapshot.hexsha) {
+    return
+  }
+  const now = new Date()
+  const age = now.getTime() - new Date(draft.modified).getTime()
+  // Upsert the retention record, resetting retention notices on hexsha change
+  let record = await DataRetention.findOne({ datasetId }).exec()
+  if (!record) {
+    record = await DataRetention.create({ datasetId, hexsha: draft.hexsha })
+  } else if (record.hexsha !== draft.hexsha) {
+    await DataRetention.updateOne(
+      { datasetId },
+      {
+        hexsha: draft.hexsha,
+        notifiedAt14Days: null,
+        notifiedAt7Days: null,
+        notifiedAtDeletion: null,
+      },
+    ).exec()
+    record = await DataRetention.findOne({ datasetId }).exec()
+  }
+  // One-time notice: no snapshot created within 24h of initial upload
+  if (
+    !lastSnapshot && age >= DAY && age < 14 * DAY && !record.notifiedNoSnapshot
+  ) {
+    await notifyWriteUsers(datasetId, (user) => ({
+      _id: `${datasetId}_${user._id}_no_snapshot_reminder`,
+      type: "email",
+      email: {
+        to: user.email,
+        name: user.name,
+        subject: "Reminder: Create a Snapshot",
+        html: snapshotReminder({
+          name: user.name,
+          datasetName: datasetId,
+          datasetId,
+          siteUrl: config.url,
+        }),
+      },
+    }))
+    await DataRetention.updateOne({ datasetId }, { notifiedNoSnapshot: now })
+      .exec()
+  }
+  // Retention warnings sent in order from 14 days, 7 days, and 0 days.
+  if (age >= 14 * DAY && !record.notifiedAt14Days) {
+    await notifyWriteUsers(datasetId, (user) => ({
+      _id: `${datasetId}_${user._id}_retention_14day`,
+      type: "email",
+      email: {
+        to: user.email,
+        name: user.name,
+        subject: "Dataset Draft Deletion Warning: 14 Days Remaining",
+        html: draftRetentionWarning({
+          name: user.name,
+          datasetId,
+          daysRemaining: 14,
+          siteUrl: config.url,
+        }),
+      },
+    }))
+    await DataRetention.updateOne({ datasetId }, { notifiedAt14Days: now })
+      .exec()
+  } else if (
+    record.notifiedAt14Days &&
+    !record.notifiedAt7Days &&
+    now.getTime() - new Date(record.notifiedAt14Days).getTime() >= 7 * DAY
+  ) {
+    await notifyWriteUsers(datasetId, (user) => ({
+      _id: `${datasetId}_${user._id}_retention_7day`,
+      type: "email",
+      email: {
+        to: user.email,
+        name: user.name,
+        subject: "Dataset Draft Deletion Warning: 7 Days Remaining",
+        html: draftRetentionWarning({
+          name: user.name,
+          datasetId,
+          daysRemaining: 7,
+          siteUrl: config.url,
+        }),
+      },
+    }))
+    await DataRetention.updateOne({ datasetId }, { notifiedAt7Days: now })
+      .exec()
+  } else if (
+    record.notifiedAt7Days &&
+    !record.notifiedAtDeletion &&
+    now.getTime() - new Date(record.notifiedAt7Days).getTime() >= 7 * DAY
+  ) {
+    await notifyWriteUsers(datasetId, (user) => ({
+      _id: `${datasetId}_${user._id}_retention_deletion`,
+      type: "email",
+      email: {
+        to: user.email,
+        name: user.name,
+        subject: "Dataset Draft Pending Deletion",
+        html: draftRetentionDeletion({
+          name: user.name,
+          datasetId,
+          siteUrl: config.url,
+        }),
+      },
+    }))
+    await DataRetention.updateOne({ datasetId }, { notifiedAtDeletion: now })
+      .exec()
+  }
+}

package/src/datalad/dataset.ts CHANGED Viewed

@@ -25,7 +25,6 @@ import Subscription from "../models/subscription"
 import BadAnnexObject from "../models/badAnnexObject"
 import { datasetsConnection } from "./pagination"
 import { getDatasetWorker } from "../libs/datalad-service"
-import notifications from "../libs/notifications"
 import { createEvent, updateEvent } from "../libs/events"
 export const giveUploaderPermission = (datasetId, userId) => {
@@ -70,7 +69,6 @@ export const createDataset = async (
     // Creation is complete here, mark successful
     await updateEvent(event)
     await subscriptions.subscribe(datasetId, uploader)
-    await notifications.snapshotReminder(datasetId)
     return ds
   } catch (e) {
     Sentry.captureException(e)

package/src/graphql/resolvers/brainInitiative.ts CHANGED Viewed

@@ -36,9 +36,14 @@ export const brainInitiative = async (
         // Fetch snapshot if metadata didn't match
         const snapshot = await latestSnapshot(dataset, null, context)
         const snapshotDescription = await description(snapshot)
-        for (const funding of snapshotDescription.Funding) {
-          if (funding.match(brainInitiativeMatch)) {
-            return true
+        if (
+          snapshotDescription?.Funding &&
+          Array.isArray(snapshotDescription.Funding)
+        ) {
+          for (const funding of snapshotDescription.Funding) {
+            if (funding.match(brainInitiativeMatch)) {
+              return true
+            }
           }
         }
         // Check for grant ids too - filter to only alphanumeric to improve matching across format differences
@@ -55,9 +60,14 @@ export const brainInitiative = async (
           ) {
             return true
           }
-          for (const funding of snapshotDescription.Funding) {
-            if (funding.replace(/[^a-zA-Z0-9]/g, "").includes(grant)) {
-              return true
+          if (
+            snapshotDescription?.Funding &&
+            Array.isArray(snapshotDescription.Funding)
+          ) {
+            for (const funding of snapshotDescription.Funding) {
+              if (funding.replace(/[^a-zA-Z0-9]/g, "").includes(grant)) {
+                return true
+              }
             }
           }
         }

package/src/graphql/utils/file.ts CHANGED Viewed

@@ -3,7 +3,7 @@ import BadAnnexObject from "../../models/badAnnexObject"
 export const filterRemovedAnnexObjects =
   (datasetId, userInfo) => async (files) => {
     const removedAnnexObjectKeys = (
-      await BadAnnexObject.find({ datasetId }).exec()
+      await BadAnnexObject.find({ datasetId, removed: true }).exec()
     ).map(({ annexKey }) => annexKey)
     // keep files that haven't had their annex objects removed
     return userInfo?.admin

package/src/libs/authentication/jwt.ts CHANGED Viewed

@@ -156,9 +156,12 @@ export const decodeJWT = (token: string): OpenNeuroTokenProfile => {
 }
 export const parsedJwtFromRequest = (req) => {
-  const jwt = jwtFromRequest(req)
-  if (jwt) return decodeJWT(jwt)
-  else return null
+  try {
+    const jwt = decodeJWT(jwtFromRequest(req))
+    return jwt || null
+  } catch (_err) {
+    return null
+  }
 }
 const refreshToken = async (jwt) => {

package/src/libs/email/templates/__tests__/__snapshots__/comment-created.spec.ts.snap CHANGED Viewed

@@ -76,7 +76,7 @@ exports[`email template -> comment created > renders with expected arguments 1`]
 		<p>
 			Sincerely,
-			The CRN Team
+			The OpenNeuro Team
 		</p>
 	</div>

package/src/libs/email/templates/__tests__/__snapshots__/dataset-deleted.spec.ts.snap CHANGED Viewed

@@ -50,7 +50,7 @@ exports[`email template -> comment created > renders with expected arguments 1`]
 		<p>
 			Sincerely,
-			The CRN Team
+			The OpenNeuro Team
 		</p>
 	</div>
 </body>

package/src/libs/email/templates/__tests__/__snapshots__/owner-unsubscribed.spec.ts.snap CHANGED Viewed

@@ -50,7 +50,7 @@ exports[`email template -> comment created > renders with expected arguments 1`]
 		<p>
 			Sincerely,
-			The CRN Team
+			The OpenNeuro Team
 		</p>
 	</div>
 </body>

package/src/libs/email/templates/__tests__/__snapshots__/snapshot-created.spec.ts.snap CHANGED Viewed

@@ -77,7 +77,7 @@ exports[`email template -> comment created > renders with expected arguments 1`]
 		<p>
 			Sincerely,
-			The CRN Team
+			The OpenNeuro Team
 		</p>
 	</div>
 </body>

package/src/libs/email/templates/__tests__/__snapshots__/snapshot-reminder.spec.ts.snap CHANGED Viewed

@@ -50,7 +50,7 @@ exports[`email template -> comment created > renders with expected arguments 1`]
 		<p>
 			Sincerely,
-			The CRN Team
+			The OpenNeuro Team
 		</p>
 		<a class="dataset-link" href="https://openneuro.org/datasets/ds12345678/snapshot">Create a snapshot. &raquo;</a>

package/src/libs/email/templates/comment-created.ts CHANGED Viewed

@@ -94,7 +94,7 @@ export const commentCreated = ({
 		<p>
 			Sincerely,
-			The CRN Team
+			The OpenNeuro Team
 		</p>
 	</div>

package/src/libs/email/templates/dataset-deleted.ts CHANGED Viewed

@@ -56,7 +56,7 @@ export const datasetDeleted = ({
 		<p>
 			Sincerely,
-			The CRN Team
+			The OpenNeuro Team
 		</p>
 	</div>
 </body>

package/src/libs/email/templates/dataset-import-failed.ts CHANGED Viewed

@@ -87,7 +87,7 @@ export const datasetImportFailed = ({
 		<p>
 			Sincerely,
-			The CRN Team
+			The OpenNeuro Team
 		</p>
 	</div>
 </body>

package/src/libs/email/templates/dataset-imported.ts CHANGED Viewed

@@ -78,7 +78,7 @@ export const datasetImportEmail = ({
 		<p>
 			Sincerely,
-			The CRN Team
+			The OpenNeuro Team
 		</p>
 	</div>
 </body>

package/src/libs/email/templates/draft-retention-deletion.ts ADDED Viewed

@@ -0,0 +1,67 @@
+export const draftRetentionDeletion = ({
+  siteUrl,
+  name,
+  datasetId,
+}: {
+  siteUrl: string
+  name: string
+  datasetId: string
+}): string =>
+  `<html>
+<head>
+<style>
+	body {
+		font-family: sans-serif;
+		font-weight: lighter;
+		background: #F5F5F5;
+	}
+	.top-bar {
+		width: 100%;
+		background: #333;
+		padding: 8px 0px 8px 15px;
+	}
+	.content {
+		padding: 15px;
+	}
+	p {
+		font-size: 16px;
+		font-weight: lighter;
+	}
+	b {
+		font-weight: bold;
+	}
+	.dataset-link {
+		display: inline-block;
+		background: #d9534f;
+		color: #FFF;
+		font-size: 20px;
+		padding: 8px 15px;
+		text-decoration: none;
+		cursor: pointer;
+	}
+</style>
+</head>
+<body>
+	<div class="top-bar">
+		<img src="${siteUrl}/assets/email-header.1cb8bf76.png" />
+	</div>
+	<div class="content">
+		<h2>Hi, ${name}</h2>
+		<p>
+			The draft data for your dataset <b>${datasetId}</b> has not been updated in some time and may be automatically removed at any time.
+		</p>
+		<p>
+			Once data has been removed you will need to re-upload your files to continue working on this dataset. Please see our data retention policy for more information: <a href="https://docs.openneuro.org/policy/data_retention.html">Data Retention Policy.</a>
+		</p>
+		<p>
+			Sincerely,
+			The OpenNeuro Team
+		</p>
+		<a class="dataset-link" href="${siteUrl}/datasets/${datasetId}">View dataset. &raquo;</a>
+	</div>
+</body>
+</html>`

package/src/libs/email/templates/draft-retention-warning.ts ADDED Viewed

@@ -0,0 +1,70 @@
+export const draftRetentionWarning = ({
+  siteUrl,
+  name,
+  datasetId,
+  daysRemaining,
+}: {
+  siteUrl: string
+  name: string
+  datasetId: string
+  daysRemaining: number
+}): string =>
+  `<html>
+<head>
+<style>
+	body {
+		font-family: sans-serif;
+		font-weight: lighter;
+		background: #F5F5F5;
+	}
+	.top-bar {
+		width: 100%;
+		background: #333;
+		padding: 8px 0px 8px 15px;
+	}
+	.content {
+		padding: 15px;
+	}
+	p {
+		font-size: 16px;
+		font-weight: lighter;
+	}
+	b {
+		font-weight: bold;
+	}
+	.dataset-link {
+		display: inline-block;
+		background: #5cb85c;
+		color: #FFF;
+		font-size: 20px;
+		padding: 8px 15px;
+		text-decoration: none;
+		cursor: pointer;
+	}
+</style>
+</head>
+<body>
+	<div class="top-bar">
+		<img src="${siteUrl}/assets/email-header.1cb8bf76.png" />
+	</div>
+	<div class="content">
+		<h2>Hi, ${name}</h2>
+		<p>
+			Your dataset <b>${datasetId}</b> has an unpublished draft that has not been updated in some time.
+			If no snapshot is created within <b>${daysRemaining} days</b>, the draft data will be automatically removed.
+		</p>
+		<p>
+			To preserve your data, please create a snapshot or update your draft. Please see our data retention policy for more information: <a href="https://docs.openneuro.org/policy/data_retention.html">Data Retention Policy.</a>
+		</p>
+		<p>
+			Sincerely,
+			The OpenNeuro Team
+		</p>
+		<a class="dataset-link" href="${siteUrl}/datasets/${datasetId}/snapshot">Create a snapshot. &raquo;</a>
+	</div>
+</body>
+</html>`

package/src/libs/email/templates/owner-unsubscribed.ts CHANGED Viewed

@@ -56,7 +56,7 @@ export const ownerUnsubscribed = ({
 		<p>
 			Sincerely,
-			The CRN Team
+			The OpenNeuro Team
 		</p>
 	</div>
 </body>

package/src/libs/email/templates/snapshot-created.ts CHANGED Viewed

@@ -89,7 +89,7 @@ export const snapshotCreated = ({
 		<p>
 			Sincerely,
-			The CRN Team
+			The OpenNeuro Team
 		</p>
 	</div>
 </body>

package/src/libs/email/templates/snapshot-reminder.ts CHANGED Viewed

@@ -58,7 +58,7 @@ export const snapshotReminder = ({
 		<p>
 			Sincerely,
-			The CRN Team
+			The OpenNeuro Team
 		</p>
 		<a class="dataset-link" href="${siteUrl}/datasets/${datasetId}/snapshot">Create a snapshot. &raquo;</a>

package/src/models/dataRetention.ts ADDED Viewed

@@ -0,0 +1,34 @@
+import mongoose from "mongoose"
+import type { Document } from "mongoose"
+const { Schema, model } = mongoose
+export interface DataRetentionDocument extends Document {
+  datasetId: string
+  // Current draft revision — when this changes, retention notices reset
+  hexsha: string
+  // Retention warning notices (reset when hexsha changes)
+  notifiedAt14Days: Date | null
+  notifiedAt7Days: Date | null
+  notifiedAtDeletion: Date | null
+  // One-time notice: no snapshot created within 24h of upload
+  notifiedNoSnapshot: Date | null
+}
+const dataRetentionSchema = new Schema({
+  datasetId: { type: String, required: true, unique: true },
+  hexsha: { type: String, required: true },
+  notifiedAt14Days: { type: Date, default: null },
+  notifiedAt7Days: { type: Date, default: null },
+  notifiedAtDeletion: { type: Date, default: null },
+  notifiedNoSnapshot: { type: Date, default: null },
+})
+/**
+ * Data retention notification status for datasets
+ */
+const DataRetention = model<DataRetentionDocument>(
+  "DataRetention",
+  dataRetentionSchema,
+)
+export default DataRetention

package/src/models/schedulerState.ts ADDED Viewed

@@ -0,0 +1,23 @@
+import mongoose from "mongoose"
+import type { Document } from "mongoose"
+const { Schema, model } = mongoose
+export interface SchedulerStateDocument extends Document {
+  key: string
+  lastRun: Date | null
+}
+const schedulerStateSchema = new Schema({
+  key: { type: String, required: true, unique: true },
+  lastRun: { type: Date, default: null },
+})
+/**
+ * Coordinates last run of any scheduled tasks across multiple server instances
+ */
+const SchedulerState = model<SchedulerStateDocument>(
+  "SchedulerState",
+  schedulerStateSchema,
+)
+export default SchedulerState

package/src/queues/consumer.ts CHANGED Viewed

@@ -2,6 +2,7 @@ import type { Consumer } from "redis-smq"
 import { reindexDataset } from "../elasticsearch/reindex-dataset"
 import { OpenNeuroQueues } from "./queues"
 import * as Sentry from "@sentry/node"
+import { checkDataRetentionNotifications } from "../datalad/dataRetentionNotifications"
 export function startConsumer(consumer: Consumer) {
   const reindexMessageHandler = async (msg, cb) => {
@@ -9,10 +10,38 @@ export function startConsumer(consumer: Consumer) {
     reindexDataset(msg.body.datasetId).then(cb)
   }
+  const dataRetentionMessageHandler = async (msg, cb) => {
+    // Check data retention and send notifications for a dataset
+    try {
+      await checkDataRetentionNotifications(msg.body.datasetId)
+      cb()
+    } catch (err) {
+      Sentry.captureException(err)
+      cb()
+    }
+  }
   consumer.consume(OpenNeuroQueues.INDEXING, reindexMessageHandler, (err) => {
     if (err) {
       Sentry.captureException(err)
     }
   })
+  consumer.consume(
+    OpenNeuroQueues.DATARETENTION,
+    dataRetentionMessageHandler,
+    (err) => {
+      if (err) {
+        Sentry.captureException(err)
+      }
+    },
+  )
+  consumer.run((err) => {
+    if (err) {
+      Sentry.captureException(err)
+    }
+  })
   return consumer
 }

package/src/queues/producer-methods.ts CHANGED Viewed

@@ -16,3 +16,17 @@ export function queueIndexDataset(datasetId: string) {
     }
   })
 }
+/**
+ * Queue search indexing for a dataset
+ * @param datasetId Dataset to index
+ */
+export function queueDataRetentionCheck(datasetId: string) {
+  const msg = new ProducibleMessage()
+  msg.setQueue(OpenNeuroQueues.DATARETENTION).setBody({ datasetId })
+  producer.produce(msg, (err) => {
+    if (err) {
+      Sentry.captureException(err)
+    }
+  })
+}

package/src/queues/queue-schedule.ts ADDED Viewed

@@ -0,0 +1,81 @@
+import * as Sentry from "@sentry/node"
+import Dataset from "../models/dataset"
+import SchedulerState from "../models/schedulerState"
+import { queueDataRetentionCheck } from "./producer-methods"
+const DAY_MS = 24 * 60 * 60 * 1000
+const POLL_INTERVAL_MS = 30 * 60 * 1000 // Check every 30 minutes
+const SCHEDULER_KEY = "dataRetentionScan"
+/**
+ * Iterate over all datasets and enqueue any per dataset actions.
+ */
+async function enqueueAllDatasetChecks(): Promise<void> {
+  const cursor = Dataset.find({}, "id").cursor()
+  for await (const dataset of cursor) {
+    // Check data retention policy status and send notifications
+    queueDataRetentionCheck(dataset.id)
+  }
+}
+/**
+ * Ensure the scheduler state document exists in MongoDB.
+ */
+async function initSchedulerState(): Promise<void> {
+  try {
+    await SchedulerState.updateOne(
+      { key: SCHEDULER_KEY },
+      { $setOnInsert: { lastRun: null } },
+      { upsert: true },
+    )
+  } catch (err) {
+    // Ignore duplicate key errors on insert race condition
+    if (!(err instanceof Error && "code" in err && err.code === 11000)) {
+      throw err
+    }
+  }
+}
+/**
+ * Attempt to atomically claim the daily data retention scan run.
+ * Returns true if this instance successfully claimed the run.
+ */
+async function claimDailyRun(): Promise<boolean> {
+  const threshold = new Date(Date.now() - DAY_MS)
+  const result = await SchedulerState.findOneAndUpdate(
+    {
+      key: SCHEDULER_KEY,
+      $or: [{ lastRun: null }, { lastRun: { $lt: threshold } }],
+    },
+    { $set: { lastRun: new Date() } },
+    { new: true },
+  )
+  return result !== null
+}
+async function runDailyCheck(): Promise<void> {
+  const claimed = await claimDailyRun()
+  if (!claimed) return
+  await enqueueAllDatasetChecks()
+}
+/**
+ * Start the daily per dataset check schedule.
+ * Polls every 30 minutes; uses a MongoDB distributed lock so only one server
+ * instance runs the scan per day, durable across restarts.
+ */
+export async function startDailySchedule(): Promise<void> {
+  const run = () => {
+    runDailyCheck().catch((err) => {
+      Sentry.captureException(err)
+      // eslint-disable-next-line no-console
+      console.error(err)
+    })
+  }
+  await initSchedulerState()
+  // Check shortly after startup, then poll every 30 minutes
+  setTimeout(run, 60 * 1000)
+  setInterval(run, POLL_INTERVAL_MS)
+}

package/src/queues/queues.ts CHANGED Viewed

@@ -1,34 +1,60 @@
 import { Queue } from "redis-smq"
 import { EQueueDeliveryModel, EQueueType, QueueRateLimit } from "redis-smq"
-import * as Sentry from "@sentry/node"
 export enum OpenNeuroQueues {
   INDEXING = "elasticsearch_indexing",
+  DATARETENTION = "data_retention",
 }
-export function setupQueues() {
-  const indexingQueue = new Queue()
-  indexingQueue.save(
-    OpenNeuroQueues.INDEXING,
-    EQueueType.FIFO_QUEUE,
-    EQueueDeliveryModel.POINT_TO_POINT,
-    (err) => {
-      // The queue may already exist, don't log that error
+function saveQueue(
+  name: string,
+  type: EQueueType,
+  delivery: EQueueDeliveryModel,
+): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const queue = new Queue()
+    queue.save(name, type, delivery, (err) => {
       if (err && err.name !== "QueueQueueExistsError") {
-        Sentry.captureException(err)
+        reject(err)
+      } else {
+        resolve()
       }
-    },
-  )
+    })
+  })
+}
-  // Limit indexing queue to 8 runs per minute to avoid stacking indexing excessively
-  const queueRateLimit = new QueueRateLimit()
-  queueRateLimit.set(
-    OpenNeuroQueues.INDEXING,
-    { limit: 8, interval: 60000 },
-    (err) => {
+function setRateLimit(
+  name: string,
+  limit: number,
+  interval: number,
+): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const queueRateLimit = new QueueRateLimit()
+    queueRateLimit.set(name, { limit, interval }, (err) => {
       if (err) {
-        Sentry.captureException(err)
+        reject(err)
+      } else {
+        resolve()
       }
-    },
+    })
+  })
+}
+export async function setupQueues(): Promise<void> {
+  await saveQueue(
+    OpenNeuroQueues.INDEXING,
+    EQueueType.FIFO_QUEUE,
+    EQueueDeliveryModel.POINT_TO_POINT,
   )
+  await saveQueue(
+    OpenNeuroQueues.DATARETENTION,
+    EQueueType.FIFO_QUEUE,
+    EQueueDeliveryModel.POINT_TO_POINT,
+  )
+  // Limit indexing queue to 8 runs per minute to avoid stacking indexing excessively
+  await setRateLimit(OpenNeuroQueues.INDEXING, 8, 60000)
+  // Rate limit data retention queue to 16 runs per minute
+  await setRateLimit(OpenNeuroQueues.DATARETENTION, 16, 60000)
 }

package/src/queues/setup.ts CHANGED Viewed

@@ -3,6 +3,7 @@ import type { IRedisSMQConfig } from "redis-smq"
 import { ERedisConfigClient } from "redis-smq-common"
 import { startConsumer } from "./consumer"
 import { setupQueues } from "./queues"
+import { startDailySchedule } from "./queue-schedule"
 import config from "../config"
 const smqConfig: IRedisSMQConfig = {
@@ -19,11 +20,21 @@ const smqConfig: IRedisSMQConfig = {
 Configuration.getSetConfig(smqConfig)
-// Producer starts automatically
 export const producer = new Producer()
 export const consumer = new Consumer()
-export function initQueues() {
-  setupQueues()
+function runProducer(): Promise<void> {
+  return new Promise((resolve, reject) => {
+    producer.run((err) => {
+      if (err) reject(err)
+      else resolve()
+    })
+  })
+}
+export async function initQueues() {
+  await setupQueues()
+  await runProducer()
   startConsumer(consumer)
+  await startDailySchedule()
 }

package/src/server.ts CHANGED Viewed

@@ -13,7 +13,7 @@ void mongoose.connect(config.mongo.url, {
 async function init() {
   // Start redis message queues
-  initQueues()
+  await initQueues()
   const app = await expressApolloSetup()
   const server = createServer(app)
   server.listen(config.port, () => {