eleventy-plugin-podcaster 2.0.0-alpha.2 → 2.0.0-alpha.3

package/.env

@@ -0,0 +1,5 @@
+BUCKET_SECRET_KEY="GiBsTUbU7VwKdSZGZKvYb47i87lWjkUXRH1Edd+I2KQ"
+BUCKET_ACCESS_KEY="DO801WKX4LJP3Y9DWE2H"
+BUCKET_ENDPOINT="https://sfo3.digitaloceanspaces.com"
+BUCKET_REGION="us-east-1"
+BUCKET_NAME="startlingbarbarabain"

package/eleventy.config.js

@@ -1,7 +1,9 @@
 import podcastFeed from './src/podcastFeed.js'
 import podcastData from './src/podcastData.js'
 import episodeData from './src/episodeData.js'
-import calculateFilenameSizeAndDuration from './src/calculateFilenameSizeAndDuration.js'
+import calculateEpisodeSizeAndDuration from './src/calculateEpisodeSizeAndDuration.js'
+import calculateEpisodeFilename from './src/calculateEpisodeFilename.js'
+
 import readableFilters from './src/readableFilters.js'
 import excerpts from './src/excerpts.js'
 import drafts from './src/drafts.js'
@@ -11,7 +13,8 @@ export default function (eleventyConfig, options = {}) {
   eleventyConfig.addPlugin(podcastFeed, options)
   eleventyConfig.addPlugin(podcastData, options)
   eleventyConfig.addPlugin(episodeData, options)
-  eleventyConfig.addPlugin(calculateFilenameSizeAndDuration, options)
+  eleventyConfig.addPlugin(calculateEpisodeSizeAndDuration, options)
+  eleventyConfig.addPlugin(calculateEpisodeFilename, options)
 
   // Filters
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "eleventy-plugin-podcaster",
-  "version": "2.0.0-alpha.2",
+  "version": "2.0.0-alpha.3",
   "description": "An Eleventy plugin that allows you to create a podcast and its accompanying website",
   "main": "eleventy.config.js",
   "exports": {
@@ -30,6 +30,7 @@
   "dependencies": {
     "@11ty/eleventy": "^3.0.0",
     "@11ty/eleventy-plugin-rss": "^2.0.1",
+    "@aws-sdk/client-s3": "^3.862.0",
     "@tsmx/human-readable": "^2.0.3",
     "chalk": "^5.3.0",
     "dom-serializer": "^2.0.0",
@@ -40,7 +41,9 @@
   },
   "devDependencies": {
     "ava": "^6.1.3",
+    "dotenv": "^17.2.1",
     "fast-xml-parser": "^4.4.0",
+    "mock-aws-s3-v3": "^6.0.5",
     "neostandard": "^0.11.4"
   }
 }

package/src/calculateEpisodeFilename.js

@@ -0,0 +1,39 @@
+function findMatchingFilename (episodeData, thisEpisode) {
+  const filenameSeasonAndEpisodePattern =
+    /^.*?\b[sS](?<seasonNumber>\d+)\s*[eE](?<episodeNumber>\d+)\b.*\.mp3$/
+  const filenameEpisodePattern = /^.*?\b(?<episodeNumber>\d+)\b.*\.mp3$/
+  const { seasonNumber, episodeNumber } = thisEpisode
+
+  for (const file of Object.keys(episodeData)) {
+    if (seasonNumber && episodeNumber) {
+      const seasonAndEpisodeMatch = file.match(filenameSeasonAndEpisodePattern)
+      if (seasonAndEpisodeMatch) {
+        const matchedSeasonNumber = parseInt(seasonAndEpisodeMatch.groups.seasonNumber)
+        const matchedEpisodeNumber = parseInt(seasonAndEpisodeMatch.groups.episodeNumber)
+        if (matchedSeasonNumber === seasonNumber &&
+            matchedEpisodeNumber === episodeNumber) {
+          return file
+        }
+      }
+    } else if (episodeNumber) {
+      const episodeMatch = file.match(filenameEpisodePattern)
+      if (episodeMatch) {
+        const matchedEpisodeNumber = parseInt(episodeMatch.groups.episodeNumber)
+        if (matchedEpisodeNumber === episodeNumber) {
+          return file
+        }
+      }
+    }
+  }
+}
+
+export default function (eleventyConfig, _options) {
+  eleventyConfig.addGlobalData('eleventyComputed.episode.filename', () => {
+    return data => {
+      if (data.episode.filename) return data.episode.filename
+      if (!data.page.inputPath.includes('/episodePosts/')) return
+
+      return findMatchingFilename(data.episodeData, data.episode)
+    }
+  })
+}

package/src/calculateEpisodeSizeAndDuration.js

@@ -0,0 +1,204 @@
+import { Duration, DateTime } from 'luxon'
+import path from 'node:path'
+import { existsSync } from 'node:fs'
+import { readdir, stat, writeFile } from 'node:fs/promises'
+import { Writable } from 'node:stream'
+import { S3Client, ListObjectsCommand, GetObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3'
+import { parseFile as parseFileMetadata, parseBuffer as parseBufferMetadata } from 'music-metadata'
+import hr from '@tsmx/human-readable'
+import chalk from 'chalk'
+
+const convertSecondsToReadableDuration = seconds =>
+  Duration.fromMillis(seconds * 1000)
+    .shiftTo('days', 'hours', 'minutes', 'seconds')
+    .toHuman()
+
+const convertReadableDurationToSeconds = duration => {
+  const durationPattern = /^(?:(?<hours>\d+):)?(?<minutes>\d{1,2}):(?<seconds>\d{2}(?:\.\d+)?)$/
+
+  let match
+  if (duration?.match) {
+    match = duration.match(durationPattern)
+  }
+
+  if (match) {
+    const hours = isNaN(parseInt(match.groups.hours))
+      ? 0
+      : parseInt(match.groups.hours)
+    const minutes = parseInt(match.groups.minutes)
+    const seconds = parseFloat(match.groups.seconds)
+    return hours * 3600 + minutes * 60 + seconds
+  }
+}
+
+async function readEpisodeDataLocally (episodeFilesDirectory) {
+  const episodes = await readdir(episodeFilesDirectory)
+  const episodeData = {}
+  for (const episode of episodes) {
+    if (!episode.endsWith('.mp3')) continue
+
+    const episodePath = path.join(episodeFilesDirectory, episode)
+    const episodeSize = (await stat(episodePath)).size
+    const episodeMetadata = await parseFileMetadata(episodePath, { duration: true })
+    const episodeDuration = episodeMetadata.format.duration
+    episodeData[episode] = {
+      size: episodeSize,
+      duration: episodeDuration
+    }
+  }
+  return episodeData
+}
+
+function calculatePodcastData (episodeData) {
+  const episodeDataValues = Object.values(episodeData)
+  const numberOfEpisodes = episodeDataValues.length
+  const totalSize = episodeDataValues.map(x => x.size).reduce((x, y) => x + y)
+  const totalDuration = episodeDataValues.map(x => x.duration).reduce((x, y) => x + y)
+  return { numberOfEpisodes, totalSize, totalDuration }
+}
+
+function reportPodcastData (podcastData) {
+  const { numberOfEpisodes, totalSize, totalDuration } = podcastData
+  console.log(chalk.yellow(`${numberOfEpisodes} episodes; ${hr.fromBytes(totalSize)}; ${convertSecondsToReadableDuration(totalDuration)}.`))
+}
+
+async function writePodcastDataLocally (episodeData, podcastData, directories) {
+  const dataDir = path.join(process.cwd(), directories.data)
+  await writeFile(path.join(dataDir, 'episodeData.json'), JSON.stringify(episodeData, null, 2))
+  await writeFile(path.join(dataDir, 'podcastData.json'), JSON.stringify(podcastData, null, 2))
+}
+
+function getS3Client (options) {
+  if (options.s3ClientObject) return options.s3ClientObject
+
+  if (options.s3Client) {
+    return new S3Client({
+      forcePathStyle: true,
+      endpoint: options.s3Client.endpoint,
+      region: options.s3Client.region,
+      credentials: {
+        accessKeyId: options.s3Client.accessKey,
+        secretAccessKey: options.s3Client.secretKey
+      }
+    })
+  }
+}
+
+async function getObjectFromS3Bucket (s3Client, s3Bucket, key) {
+  const getObjectResponse = await s3Client.send(new GetObjectCommand({ Bucket: s3Bucket, Key: key }))
+
+  const chunks = []
+  if (typeof getObjectResponse.Body.pipe === 'function') {
+    // this is to cope with the behaviour of the mock, which doesn't return an iterator full of chunks
+    const writable = new Writable({
+      write (chunk, encoding, callback) {
+        chunks.push(chunk)
+        callback()
+      },
+      final (callback) {
+        callback()
+      }
+    })
+    getObjectResponse.Body.pipe(writable)
+    await new Promise((resolve, reject) => {
+      writable.on('finish', resolve)
+      writable.on('error', reject)
+    })
+  } else {
+    for await (const chunk of getObjectResponse.Body) {
+      chunks.push(chunk)
+    }
+  }
+  const buffer = Buffer.concat(chunks)
+  return { buffer, lastModified: getObjectResponse.LastModified }
+}
+
+async function getStoredEpisodeDataFromS3Bucket (s3Client, s3BucketName) {
+  try {
+    const { buffer, lastModified } = await getObjectFromS3Bucket(s3Client, s3BucketName, 'episodeData.json')
+    return { episodeData: JSON.parse(buffer.toString()), lastModified }
+  } catch (err) {
+    return { episodeData: {}, lastModified: null }
+  }
+}
+
+async function updateEpisodeDataFromS3Bucket (s3Client, s3Bucket) {
+  const storedEpisodeData = await getStoredEpisodeDataFromS3Bucket(s3Client, s3Bucket)
+  const storedEpisodeDataLastModifiedDate = (storedEpisodeData.lastModified)
+    ? DateTime.fromISO(storedEpisodeData.lastModified)
+    : null
+  const list = await s3Client.send(new ListObjectsCommand({ Bucket: s3Bucket }))
+  const result = { ...storedEpisodeData.episodeData }
+  for (const item of list.Contents ?? []) {
+    if (!item.Key.endsWith('.mp3')) continue
+
+    const { Key: filename, Size: size, LastModified: lastModified } = item
+
+    if (!(filename in result) ||
+        !('size' in result[filename]) ||
+        !('duration' in result[filename]) ||
+        !storedEpisodeDataLastModifiedDate ||
+        storedEpisodeDataLastModifiedDate > DateTime.fromISO(lastModified)) {
+      const { buffer } = await getObjectFromS3Bucket(s3Client, s3Bucket, filename)
+      const metadata = await parseBufferMetadata(buffer, null, { duration: true })
+      const duration = metadata.format.duration
+      result[filename] = { size, duration }
+    }
+  }
+  return result
+}
+
+async function writeEpisodeDataToS3Bucket (s3Client, s3Bucket, episodeData) {
+  await s3Client.send(new PutObjectCommand({
+    Bucket: s3Bucket,
+    Key: 'episodeData.json',
+    Body: JSON.stringify(episodeData, null, 2),
+    ContentType: 'application/json'
+  }))
+}
+
+export default function (eleventyConfig, options = {}) {
+  let firstRun = true
+  eleventyConfig.on('eleventy.before', async ({ directories }) => {
+    if (!firstRun || process.env.SKIP_EPISODE_CALCULATIONS === 'true') return
+    firstRun = false
+
+    const episodeFilesDirectory = path.join(directories.input, 'episodeFiles')
+    let episodeData
+    if (existsSync(episodeFilesDirectory)) {
+      episodeData = await readEpisodeDataLocally(episodeFilesDirectory)
+    } else if (options.s3ClientObject || options.s3Client) {
+      const s3Client = getS3Client(options)
+      const s3Bucket = options.s3Client.bucket
+      episodeData = await updateEpisodeDataFromS3Bucket(s3Client, s3Bucket)
+      await writeEpisodeDataToS3Bucket(s3Client, s3Bucket, episodeData)
+    } else {
+      return
+    }
+    const podcastData = calculatePodcastData(episodeData)
+    await writePodcastDataLocally(episodeData, podcastData, directories)
+    if (!eleventyConfig.quietMode) reportPodcastData(podcastData)
+  })
+
+  eleventyConfig.addGlobalData('eleventyComputed.episode.size', () => {
+    return data => {
+      if (data.episode.size) return data.episode.size
+      if (data.page.inputPath.includes('/episodePosts/') && data.episodeData) {
+        return data.episodeData[data.episode.filename]?.size
+      }
+    }
+  })
+
+  eleventyConfig.addGlobalData('eleventyComputed.episode.duration', () => {
+    return data => {
+      if (data.episode.duration) {
+        const convertedReadableDuration = convertReadableDurationToSeconds(data.episode.duration)
+        return convertedReadableDuration ?? data.episode.duration
+      }
+
+      if (data.page.inputPath.includes('/episodePosts/') && data.episodeData) {
+        return data.episodeData[data.episode.filename]?.duration
+      }
+    }
+  })
+}

package/src/drafts.js

@@ -13,9 +13,9 @@ export default (eleventyConfig, options = {}) => {
     }
     if (!hasLoggedAboutDrafts) {
       if (shouldIncludeDrafts) {
-        console.log('Including drafts.')
+        if (!eleventyConfig.quietMode) console.log('Including drafts.')
       } else {
-        console.log('Excluding drafts.')
+        if (!eleventyConfig.quietMode) console.log('Excluding drafts.')
       }
       hasLoggedAboutDrafts = true
     }

package/src/calculateFilenameSizeAndDuration.js

@@ -1,106 +0,0 @@
-import { Duration } from 'luxon'
-import path from 'node:path'
-import { existsSync } from 'node:fs'
-import { readdir, stat, writeFile } from 'node:fs/promises'
-import { parseFile } from 'music-metadata'
-import hr from '@tsmx/human-readable'
-import chalk from 'chalk'
-
-const convertSecondsToReadableDuration = seconds =>
-  Duration.fromMillis(seconds * 1000)
-    .shiftTo('days', 'hours', 'minutes', 'seconds')
-    .toHuman()
-
-export default function (eleventyConfig) {
-  let firstRun = true
-  eleventyConfig.on('eleventy.before', async ({ directories }) => {
-    // don't keep recalculating episode data in serve mode
-    if (!firstRun || process.env.SKIP_EPISODE_CALCULATIONS === 'true') return
-    firstRun = false
-    const episodesDir = path.join(directories.input, 'episodeFiles')
-    if (!existsSync(episodesDir)) return
-
-    const episodes = await readdir(episodesDir)
-    const episodeData = {}
-    let numberOfEpisodes = 0
-    let totalSize = 0
-    let totalDuration = 0
-
-    for (const episode of episodes) {
-      if (!episode.endsWith('.mp3')) continue
-
-      numberOfEpisodes++
-      const episodePath = path.join(episodesDir, episode)
-      const episodeSize = (await stat(episodePath)).size
-      totalSize += episodeSize
-      const episodeMetadata = await parseFile(episodePath, { duration: true })
-      const episodeDuration = episodeMetadata.format.duration
-      totalDuration += episodeDuration
-      episodeData[episode] = {
-        size: episodeSize,
-        duration: Math.round(episodeDuration * 1000) / 1000
-      }
-      totalDuration = Math.round(totalDuration * 1000) / 1000
-    }
-    const podcastData = { numberOfEpisodes, totalSize, totalDuration }
-
-    const dataDir = path.join(process.cwd(), directories.data)
-    await writeFile(path.join(dataDir, 'episodeData.json'), JSON.stringify(episodeData, null, 2))
-    await writeFile(path.join(dataDir, 'podcastData.json'), JSON.stringify(podcastData, null, 2))
-
-    console.log(chalk.yellow(`${numberOfEpisodes} episodes; ${hr.fromBytes(totalSize)}; ${convertSecondsToReadableDuration(totalDuration)}.`))
-  })
-
-  const filenameSeasonAndEpisodePattern =
-    /^.*?\b[sS](?<seasonNumber>\d+)\s*[eE](?<episodeNumber>\d+)\b.*\.mp3$/
-  const filenameEpisodePattern = /^.*?\b(?<episodeNumber>\d+)\b.*\.mp3$/
-
-  eleventyConfig.addGlobalData('eleventyComputed.episode.filename', () => {
-    return data => {
-      if (data.episode.filename) return data.episode.filename
-
-      if (!data.page.inputPath.includes('/episodePosts/')) return
-
-      for (const file of Object.keys(data.episodeData)) {
-        if (data.episode.seasonNumber && data.episode.episodeNumber) {
-          const seasonAndEpisodeMatch = file.match(filenameSeasonAndEpisodePattern)
-          if (seasonAndEpisodeMatch) {
-            const matchedSeasonNumber = parseInt(seasonAndEpisodeMatch.groups.seasonNumber)
-            const matchedEpisodeNumber = parseInt(seasonAndEpisodeMatch.groups.episodeNumber)
-            if (matchedSeasonNumber === data.episode.seasonNumber &&
-                matchedEpisodeNumber === data.episode.episodeNumber) {
-              return file
-            }
-          }
-        } else if (data.episode.episodeNumber) {
-          const episodeMatch = file.match(filenameEpisodePattern)
-          if (episodeMatch) {
-            const matchedEpisodeNumber = parseInt(episodeMatch.groups.episodeNumber)
-            if (matchedEpisodeNumber === data.episode.episodeNumber) {
-              return file
-            }
-          }
-        }
-      }
-    }
-  })
-
-  eleventyConfig.addGlobalData('eleventyComputed.episode.size', () => {
-    return data => {
-      if (data.episode.size) return data.episode.size
-      if (data.page.inputPath.includes('/episodePosts/') && data.episodeData) {
-        return data.episodeData[data.episode.filename]?.size
-      }
-    }
-  })
-
-  eleventyConfig.addGlobalData('eleventyComputed.episode.duration', () => {
-    return data => {
-      if (data.episode.duration) return data.episode.duration
-
-      if (data.page.inputPath.includes('/episodePosts/') && data.episodeData) {
-        return data.episodeData[data.episode.filename]?.duration
-      }
-    }
-  })
-}