ffxiv_ptfinder 1.0.1

package/fetch-content.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+BASE="https://v2.xivapi.com/api/sheet/ContentFinderCondition"
+after=0
+
+TMP_NDJSON="duties_all.tmp.ndjson"
+OUT_JSON="duties_all.json"
+
+# 一旦 tmp を消す
+rm -f "$TMP_NDJSON"
+
+echo "Fetching duties from XIVAPI..."
+
+while :; do
+  echo "  - after=${after} で取得中..."
+  res=$(curl -s "${BASE}?fields=Name&language=ja&limit=200&after=${after}")
+
+  # このページの rows を NDJSON 形式で追記
+  echo "$res" \
+    | jq -c '.rows[] | select(.fields.Name != "") | {duty_id: .row_id, duty_name: .fields.Name}' \
+    >> "$TMP_NDJSON"
+
+  # 取れた件数確認
+  count=$(echo "$res" | jq '.rows | length')
+
+  # 200件未満になったら最後のページ
+  if [ "$count" -lt 200 ]; then
+    break
+  fi
+
+  # 次ページ用の after を最後の row_id に更新
+  after=$(echo "$res" | jq '.rows[-1].row_id')
+done
+
+echo "Building pretty JSON..."
+
+# NDJSON を配列にまとめて、キーを duty_id 順にソートして整形出力
+jq -s 'sort_by(.duty_id)' "$TMP_NDJSON" > "$OUT_JSON"
+
+# tmp は削除
+rm -f "$TMP_NDJSON"
+
+echo "✅ Done: ${OUT_JSON}"

package/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "ffxiv_ptfinder",
+  "version": "1.0.1",
+  "license": "MIT",
+  "main": "dist/cli.js",
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "start": "tsx src/debugCli.ts",
+    "dev": "tsx watch src/debugCli.ts",
+    "notify": "tsx src/notifyCli.ts",
+    "sam:build": "SAM_CLI_HOME=.samcli sam build",
+    "sam:deploy": "SAM_CLI_HOME=.samcli sam deploy --guided"
+  },
+  "dependencies": {
+    "@piyoraik/ffxiv-lodestone-character-lookup": "^1.0.0",
+    "axios": "^1.7.9",
+    "cheerio": "^1.1.2"
+  },
+  "devDependencies": {
+    "@types/node": "^22.10.2",
+    "tsx": "^4.20.0",
+    "typescript": "^5.7.2"
+  }
+}

package/samconfig.toml

@@ -0,0 +1,13 @@
+version = 0.1
+
+[default.deploy.parameters]
+stack_name = "ffxiv-finder-ultima"
+resolve_s3 = true
+s3_prefix = "ffxiv-finder-ultima"
+region = "ap-northeast-1"
+capabilities = "CAPABILITY_IAM"
+parameter_overrides = "Limit=\"5\" FilterFile=\"data/filter.json\""
+image_repositories = []
+
+[default.global.parameters]
+region = "ap-northeast-1"

package/src/cliOptions.ts

@@ -0,0 +1,79 @@
+import { DEFAULT_LISTINGS_URL, DEFAULT_WEBHOOK_LIMIT, ENV } from "./config";
+
+export type CliOptions = {
+  query?: string;
+  limit?: number;
+  output?: "json" | "discord";
+};
+
+/**
+ * Usage を stderr に出力して終了します。
+ */
+export function printUsageAndExit(exitCode = 0): never {
+  process.stderr.write(getUsageText() + "\n");
+  process.exit(exitCode);
+}
+
+/**
+ * Usage 文字列を生成します（テストしやすいように純粋関数にしています）。
+ */
+export function getUsageText(): string {
+  return [
+    "Usage:",
+    `  yarn start -- [--limit <n>] [--discord]`,
+    `    defaults: --limit ${DEFAULT_WEBHOOK_LIMIT}`,
+    "",
+    "Environment variables (Lambda-friendly):",
+    `  ${ENV.LIMIT}=<n>  (default: ${DEFAULT_WEBHOOK_LIMIT})`,
+    `  ${ENV.DISCORD_WEBHOOK_URL}=<discord_webhook_url>  (required)`,
+    `  ${ENV.FILTER_FILE}=<path>  (default: data/filter.json)`,
+    "Examples:",
+    `  ${ENV.DISCORD_WEBHOOK_URL}=\"https://discord.com/api/webhooks/...\" yarn notify`,
+    `  ${ENV.LIMIT}=5 ${ENV.DISCORD_WEBHOOK_URL}=\"https://discord.com/api/webhooks/...\" yarn notify`,
+    `  yarn start -- --discord --limit 5`
+  ].join("\n");
+}
+
+/**
+ * CLI 引数（argv）をオプションにパースします。
+ * ここでは argv の解析のみ行い、環境変数とのマージやデフォルト適用は別モジュールで行います。
+ */
+export function parseCliArgs(argv: string[]): CliOptions {
+  const options: CliOptions = {};
+
+  for (let index = 0; index < argv.length; index++) {
+    const arg = argv[index];
+
+    if (arg === "--help" || arg === "-h") {
+      printUsageAndExit(0);
+    }
+
+    if (arg === "--query" || arg === "-q") {
+      options.query = argv[++index];
+      continue;
+    }
+
+    if (arg === "--discord") {
+      options.output = "discord";
+      continue;
+    }
+
+    if (arg === "--limit" || arg === "-l") {
+      const raw = argv[++index];
+      const parsed = Number(raw);
+      if (!Number.isFinite(parsed) || parsed <= 0) {
+        process.stderr.write(`Invalid --limit: ${raw}\n`);
+        process.exit(2);
+      }
+      options.limit = Math.floor(parsed);
+      continue;
+    }
+
+    if (!options.query) {
+      options.query = arg;
+      continue;
+    }
+  }
+
+  return options;
+}

package/src/config.ts

@@ -0,0 +1,16 @@
+/**
+ * CLI 全体で共有するデフォルト値・設定。
+ */
+
+export const DEFAULT_LISTINGS_URL = "https://xivpf.com/listings";
+export const DEFAULT_COOKIE = "lang=ja";
+export const DEFAULT_WEBHOOK_LIMIT = 5;
+export const DEFAULT_FILTER_FILE = "data/filter.json";
+
+export const ALLOWED_DATA_CENTRES = new Set(["Elemental", "Mana", "Meteor", "Gaia"]);
+
+export const ENV = {
+  LIMIT: "FFXIV_PTFINDER_LIMIT",
+  DISCORD_WEBHOOK_URL: "DISCORD_WEBHOOK_URL",
+  FILTER_FILE: "FFXIV_PTFINDER_FILTER_FILE"
+} as const;

package/src/debugCli.ts

@@ -0,0 +1,124 @@
+import { DEFAULT_FILTER_FILE, DEFAULT_LISTINGS_URL } from "./config";
+import { parseCliArgs } from "./cliOptions";
+import { readEnvOptions } from "./envOptions";
+import { loadJobsJa } from "./jobs";
+import { buildListings } from "./listingsPipeline";
+import { toDiscordCodeBlock } from "./discordWebhook";
+import { formatListingText, type PartyRole } from "./partyText";
+import { buildJobMaps } from "./jobMaps";
+import { createLodestoneEnrichmentCache, enrichListingWithLodestone } from "./lodestoneEnrichment";
+import { loadListingSearchFilterFromFile, matchListing } from "./searchFilter";
+import type { ListingSearchFilter } from "./searchFilter";
+import type { Listing } from "./types";
+
+/**
+ * パイプ出力時（例: `| head`）の EPIPE を無視するハンドラを登録します。
+ */
+function installStdoutEpipeHandler(): void {
+  process.stdout.on("error", (err: NodeJS.ErrnoException) => {
+    if (err.code === "EPIPE") process.exit(0);
+    throw err;
+  });
+}
+
+/**
+ * デバッグ用エントリポイント。
+ *
+ * - Webhook 送信はせず、募集一覧を JSON で標準出力に吐き出します。
+ * - 入力は固定で `https://xivpf.com/listings` を使用します。
+ */
+async function main(): Promise<void> {
+  installStdoutEpipeHandler();
+
+  const cli = parseCliArgs(process.argv.slice(2));
+  const env = readEnvOptions();
+  const filterFile = env.filterFile ?? DEFAULT_FILTER_FILE;
+  const searchFilter = await loadListingSearchFilterFromFile(filterFile);
+
+  const preFiltered = await buildListings({
+    input: DEFAULT_LISTINGS_URL,
+    searchFilter
+  });
+
+  const limit = cli.limit ?? env.limit;
+  const jobsJa = await loadJobsJa();
+  const { codeToShort, codeToRole } = buildJobMaps(jobsJa.jobs);
+
+  const listings = await applyFullFilter({
+    listings: preFiltered,
+    searchFilter,
+    limit,
+    codeToShort,
+    codeToRole,
+    enrichLodestone: cli.output === "discord" || Boolean(searchFilter?.achievements)
+  });
+
+  if (cli.output === "discord") {
+    process.stdout.write(formatDiscordPreview(listings, codeToShort, codeToRole) + "\n");
+    return;
+  }
+
+  process.stdout.write(JSON.stringify(listings, null, 2) + "\n");
+}
+
+/**
+ * `filter.json` の内容をできるだけ実運用（Lambda）に近い形で評価します。
+ *
+ * - party/formattedText はここで評価します（`buildListings` の事前フィルタには含まれないため）
+ * - achievements 指定がある場合は、Lodestoneアクセスを行い達成状況を補完します
+ */
+async function applyFullFilter(params: {
+  listings: Listing[];
+  searchFilter: ListingSearchFilter | undefined;
+  limit: number | undefined;
+  codeToShort: Map<string, string>;
+  codeToRole: Map<string, PartyRole>;
+  enrichLodestone: boolean;
+}): Promise<Listing[]> {
+  const hasSearchFilter = Boolean(params.searchFilter);
+  if (!hasSearchFilter) return params.limit ? params.listings.slice(0, params.limit) : params.listings;
+
+  const lodestoneCache = params.enrichLodestone ? createLodestoneEnrichmentCache() : undefined;
+
+  const matched: Listing[] = [];
+  for (const listing of params.listings) {
+    const enriched =
+      params.enrichLodestone && lodestoneCache
+        ? await enrichListingWithLodestone(listing, lodestoneCache)
+        : listing;
+
+    const formattedText = formatListingText(enriched, params.codeToShort, params.codeToRole);
+    const ok = await matchListing({
+      listing: enriched,
+      filter: params.searchFilter,
+      formattedText
+    });
+    if (!ok) continue;
+
+    matched.push(enriched);
+    if (params.limit && matched.length >= params.limit) break;
+  }
+
+  return matched;
+}
+
+/**
+ * Webhook送信の本文と同じ形式（コードブロック）で標準出力に出します。
+ *
+ * - 1募集=1メッセージ想定なので、募集ごとにコードブロックを出します
+ * - ヒット0件の場合は、Webhookで送る文言と同じ文言を出します
+ */
+function formatDiscordPreview(
+  listings: Listing[],
+  codeToShort: Map<string, string>,
+  codeToRole: Map<string, PartyRole>
+): string {
+  if (listings.length === 0) return "該当の募集は見つかりませんでした";
+  return listings.map((l) => toDiscordCodeBlock(formatListingText(l, codeToShort, codeToRole))).join("\n\n");
+}
+
+main().catch((err: unknown) => {
+  const message = err instanceof Error ? err.stack ?? err.message : String(err);
+  process.stderr.write(message + "\n");
+  process.exitCode = 1;
+});

package/src/descriptionRequirements.ts

@@ -0,0 +1,36 @@
+import type { Listing } from "./types";
+
+/**
+ * 表示用に空白を正規化します。
+ */
+export function normalizeSpaces(value: string): string {
+  return value.replace(/\s+/g, " ").trim();
+}
+
+/**
+ * description からタグ（例: `[Practice]`）を検出してラベルを抽出し、
+ * 本文からタグトークンを除去します。
+ *
+ * `Listing.description` を読みやすく保ち、タグの意味は `Listing.requirements` に移します。
+ */
+export function applyDescriptionTags(
+  listings: Listing[],
+  tags: Array<{ token: string; label: string }>
+): Listing[] {
+  return listings.map((listing) => {
+    const requirements = new Set<string>();
+    let description = listing.description;
+
+    for (const tag of tags) {
+      if (!tag.token) continue;
+      if (description.includes(tag.token)) requirements.add(tag.label);
+      description = description.split(tag.token).join("");
+    }
+
+    return {
+      ...listing,
+      requirements: Array.from(requirements),
+      description: normalizeSpaces(description)
+    };
+  });
+}

package/src/descriptionTags.ts

@@ -0,0 +1,20 @@
+import { readFile } from "node:fs/promises";
+import { resolve } from "node:path";
+
+export type DescriptionTagsJa = {
+  version: number;
+  tags: Array<{ token: string; label: string }>;
+};
+
+let cachedTags: DescriptionTagsJa | undefined;
+
+/**
+ * `data/description_tags_ja.json` を読み込みます（初回のみ読み取り、以後はキャッシュ）。
+ */
+export async function loadDescriptionTagsJa(): Promise<DescriptionTagsJa> {
+  if (cachedTags) return cachedTags;
+  const filePath = resolve(process.cwd(), "data/description_tags_ja.json");
+  const raw = await readFile(filePath, "utf8");
+  cachedTags = JSON.parse(raw) as DescriptionTagsJa;
+  return cachedTags;
+}

package/src/discordWebhook.ts

@@ -0,0 +1,49 @@
+import axios from "axios";
+
+/**
+ * 指定ミリ秒だけ待機します。
+ */
+export async function sleep(ms: number): Promise<void> {
+  await new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Discord のコードブロック形式に変換し、2000文字制限に引っかからないように切り詰めます。
+ */
+export function toDiscordCodeBlock(text: string, maxChars = 1900): string {
+  const sanitized = text.replace(/```/g, "'''").trim();
+  const truncated =
+    sanitized.length > maxChars ? sanitized.slice(0, maxChars - 1).trimEnd() + "…" : sanitized;
+  return "```\n" + truncated + "\n```";
+}
+
+async function sendWebhook(webhookUrl: string, content: string): Promise<void> {
+  await axios.post(
+    webhookUrl,
+    { content },
+    {
+      headers: { "Content-Type": "application/json" },
+      timeout: 30_000
+    }
+  );
+}
+
+/**
+ * Discord Webhook に送信します（レート制限: HTTP 429 の場合は 1 回だけリトライします）。
+ */
+export async function postDiscordWebhook(webhookUrl: string, content: string): Promise<void> {
+  try {
+    await sendWebhook(webhookUrl, content);
+    return;
+  } catch (err: unknown) {
+    if (!axios.isAxiosError(err)) throw err;
+    if (err.response?.status !== 429) throw err;
+
+    const retryAfterMs =
+      typeof err.response.data?.retry_after === "number"
+        ? Math.ceil(err.response.data.retry_after * 1000)
+        : 1500;
+    await sleep(retryAfterMs);
+    await sendWebhook(webhookUrl, content);
+  }
+}

package/src/envOptions.ts

@@ -0,0 +1,72 @@
+import {
+  DEFAULT_FILTER_FILE,
+  DEFAULT_LISTINGS_URL,
+  DEFAULT_WEBHOOK_LIMIT,
+  ENV
+} from "./config";
+import type { CliOptions } from "./cliOptions";
+import { type ListingSearchFilter } from "./searchFilter";
+
+/**
+ * 環境変数を読み取り、前後空白を除いた文字列を返します（空なら undefined）。
+ */
+export function getEnvString(name: string): string | undefined {
+  const value = process.env[name];
+  const trimmed = typeof value === "string" ? value.trim() : "";
+  return trimmed.length > 0 ? trimmed : undefined;
+}
+
+/**
+ * 正の整数として解釈できる場合のみ数値を返します。
+ */
+export function parsePositiveInt(value: string | undefined): number | undefined {
+  if (!value) return undefined;
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed) || parsed <= 0) return undefined;
+  return Math.floor(parsed);
+}
+
+/**
+ * 環境変数からオプションを読み取ります（デフォルト値はここでは適用しません）。
+ */
+export function readEnvOptions(): Partial<CliOptions> & {
+  filterFile?: string;
+} {
+  return {
+    limit: parsePositiveInt(getEnvString(ENV.LIMIT)),
+    filterFile: getEnvString(ENV.FILTER_FILE)
+  };
+}
+
+export type ResolvedCliOptions = {
+  input: string;
+  webhookUrl: string;
+  limit: number;
+  searchFilter?: ListingSearchFilter;
+  searchFilterFile: string;
+};
+
+/**
+ * CLIオプションを優先して環境変数オプションとマージし、デフォルト値を適用します。
+ */
+export function resolveOptions(
+  cli: CliOptions,
+  env: Partial<CliOptions> & {
+    filterFile?: string;
+  }
+): ResolvedCliOptions {
+  const webhookUrl = getEnvString(ENV.DISCORD_WEBHOOK_URL);
+  if (!webhookUrl) {
+    throw new Error(`${ENV.DISCORD_WEBHOOK_URL} is required.`);
+  }
+
+  const searchFilterFile = env.filterFile ?? DEFAULT_FILTER_FILE;
+
+  return {
+    input: DEFAULT_LISTINGS_URL,
+    webhookUrl,
+    limit: cli.limit ?? env.limit ?? DEFAULT_WEBHOOK_LIMIT,
+    searchFilter: undefined,
+    searchFilterFile
+  };
+}

package/src/extractListings.ts

@@ -0,0 +1,121 @@
+import * as cheerio from "cheerio";
+import type { CheerioAPI } from "cheerio";
+import type { Listing, PartyItem } from "./types";
+
+/**
+ * HTML から取り出したテキストを、出力用に読みやすく正規化します。
+ */
+function normalizeText(text: string): string {
+  return text
+    .replace(/\r\n/g, "\n")
+    .replace(/[ \t]+/g, " ")
+    .replace(/\n[ \t]+/g, "\n")
+    .trim();
+}
+
+/**
+ * class 属性（スペース区切り）を配列に分解します。
+ */
+function splitClassList(value: string | undefined): string[] {
+  return (value ?? "").split(/\s+/).map((s) => s.trim()).filter(Boolean);
+}
+
+/**
+ * 募集一覧の要素（`.listing[data-id]`）を取得します。
+ * 可能なら `#listings` 配下から取り、なければドキュメント全体から探索します。
+ */
+function selectListingElements($: CheerioAPI) {
+  const listingsRoot = $("#listings");
+  return listingsRoot.length
+    ? listingsRoot.find("div.listing[data-id]")
+    : $("div.listing[data-id]");
+}
+
+/**
+ * `.listing` 要素から duty 情報（class と title）を抽出します。
+ */
+function parseDuty(listing: cheerio.Cheerio<any>): Listing["duty"] | undefined {
+  const dutyNode = listing.find(".duty").first();
+  if (!dutyNode.length) return undefined;
+
+  const text = normalizeText(dutyNode.text());
+  const title = (dutyNode.attr("title") ?? text).trim();
+  if (!title) return undefined;
+
+  return { classList: splitClassList(dutyNode.attr("class")), title };
+}
+
+/**
+ * `.listing` 要素から description テキストを抽出します。
+ */
+function parseDescription(listing: cheerio.Cheerio<any>): string {
+  const descriptionText = listing.find(".description").first().text();
+  return normalizeText(descriptionText);
+}
+
+/**
+ * `.listing` 要素から募集者（creator）を抽出します。
+ */
+function parseCreator(listing: cheerio.Cheerio<any>): string | undefined {
+  const creatorText = listing.find(".right.meta .item.creator .text").first().text();
+  const normalized = normalizeText(creatorText);
+  return normalized.length > 0 ? normalized : undefined;
+}
+
+/**
+ * `.listing` 要素から party 情報（各 slot の class と title）を抽出します。
+ */
+function parseParty($: CheerioAPI, listing: cheerio.Cheerio<any>): PartyItem[] {
+  const party: PartyItem[] = [];
+  const partyRoot = listing.find(".party").first();
+  if (!partyRoot.length) return party;
+
+  partyRoot.children("div").each((_, child) => {
+    const node = $(child);
+    const classList = splitClassList(node.attr("class"));
+    if (classList.length === 0) return;
+    const title = node.attr("title") ?? undefined;
+    party.push(title ? { classList, title } : { classList });
+  });
+
+  return party;
+}
+
+/**
+ * `.listing` 要素を `Listing` に変換します。
+ */
+function parseListingElement($: CheerioAPI, el: unknown): Listing | undefined {
+  const listing = $(el as any);
+  const id = listing.attr("data-id");
+  if (!id) return undefined;
+
+  return {
+    id,
+    dataCentre: listing.attr("data-centre") ?? undefined,
+    dataPfCategory: listing.attr("data-pf-category") ?? undefined,
+    duty: parseDuty(listing),
+    creator: parseCreator(listing),
+    requirements: [],
+    description: parseDescription(listing),
+    party: parseParty($, listing)
+  };
+}
+
+/**
+ * listings HTML から募集一覧を抽出します。
+ */
+export function extractListingsFromHtml(html: string): Listing[] {
+  const $ = cheerio.load(html);
+
+  const results: Listing[] = [];
+  const seenIds = new Set<string>();
+  selectListingElements($).each((_, el) => {
+    const parsed = parseListingElement($, el);
+    if (!parsed) return;
+    if (seenIds.has(parsed.id)) return;
+    seenIds.add(parsed.id);
+    results.push(parsed);
+  });
+
+  return results;
+}

package/src/fetchHtml.ts

@@ -0,0 +1,41 @@
+import axios from "axios";
+import { readFile } from "node:fs/promises";
+import { DEFAULT_COOKIE, DEFAULT_LISTINGS_URL } from "./config";
+
+/**
+ * 入力が HTTP(S) URL っぽい場合に true を返します。
+ */
+export function isHttpUrl(value: string): boolean {
+  return /^https?:\/\//i.test(value);
+}
+
+/**
+ * HTML をローカルファイル、または URL から読み込みます。
+ *
+ * URL の場合は `Cookie: lang=ja` を付与します。
+ * ※ Lambda(Node.js) の CommonJS から ESM 専用ライブラリを require できない問題を避けるため、
+ *   CookieJar は使わずヘッダで渡します（xivpf は通常リダイレクトしない想定）。
+ */
+export async function loadHtml(input: string): Promise<string> {
+  if (!isHttpUrl(input)) {
+    return await readFile(input, "utf8");
+  }
+
+  const response = await axios.get<string>(input, {
+    responseType: "text",
+    maxRedirects: 5,
+    beforeRedirect: (options) => {
+      options.headers = options.headers ?? {};
+      options.headers["Cookie"] = DEFAULT_COOKIE;
+      options.headers["Accept-Language"] = "ja,en;q=0.8";
+    },
+    headers: {
+      Accept: "text/html,application/xhtml+xml",
+      "Accept-Language": "ja,en;q=0.8",
+      Cookie: DEFAULT_COOKIE,
+      "User-Agent": "ffxiv_ptfinder/1.0 (+https://xivpf.com)"
+    }
+  });
+
+  return response.data;
+}

package/src/jobMaps.ts

@@ -0,0 +1,16 @@
+import type { PartyRole } from "./partyText";
+
+/**
+ * party の整形に使うジョブ変換マップを作成します。
+ */
+export function buildJobMaps(jobs: Record<string, { short: string; role: PartyRole }>): {
+  codeToShort: Map<string, string>;
+  codeToRole: Map<string, PartyRole>;
+} {
+  const entries = Object.entries(jobs);
+  return {
+    codeToShort: new Map(entries.map(([code, info]) => [code, info.short])),
+    codeToRole: new Map(entries.map(([code, info]) => [code, info.role]))
+  };
+}
+

package/src/jobs.ts

@@ -0,0 +1,20 @@
+import { readFile } from "node:fs/promises";
+import { resolve } from "node:path";
+
+export type JobsJa = {
+  version: number;
+  jobs: Record<string, { short: string; role: "tank" | "healer" | "dps" }>;
+};
+
+let cachedJobs: JobsJa | undefined;
+
+/**
+ * `data/jobs_ja.json` を読み込みます（初回のみ読み取り、以後はキャッシュ）。
+ */
+export async function loadJobsJa(): Promise<JobsJa> {
+  if (cachedJobs) return cachedJobs;
+  const filePath = resolve(process.cwd(), "data/jobs_ja.json");
+  const raw = await readFile(filePath, "utf8");
+  cachedJobs = JSON.parse(raw) as JobsJa;
+  return cachedJobs;
+}