disqus-wxr-middleware 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +202 -0
- package/bin/disqus-wxr.js +12 -0
- package/package.json +49 -0
- package/src/checkpoint.js +182 -0
- package/src/cli.js +110 -0
- package/src/config.js +69 -0
- package/src/disqusClient.js +283 -0
- package/src/index.js +113 -0
- package/src/logger.js +45 -0
- package/src/normalize.js +38 -0
- package/src/pipeline.js +290 -0
- package/src/statusMap.js +33 -0
- package/src/threadCache.js +66 -0
- package/src/wxrMapping.js +119 -0
- package/src/wxrWriter.js +160 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Semiologic
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
# disqus-wxr-middleware
|
|
2
|
+
|
|
3
|
+
[](https://github.com/graphcomment/disqus-wxr-middleware/actions/workflows/ci.yml)
|
|
4
|
+
[](./LICENSE)
|
|
5
|
+
|
|
6
|
+
> Exports a **Disqus** forum to a **WordPress WXR** file through the Disqus API.
|
|
7
|
+
> A standalone command-line tool, run by the client with their own Disqus
|
|
8
|
+
> credentials — **no dependency on any third-party infrastructure**.
|
|
9
|
+
|
|
10
|
+
The generated WXR file is a standard WordPress export (WXR 1.2): it can be imported
|
|
11
|
+
as-is into GraphComment through the WordPress import flow.
|
|
12
|
+
|
|
13
|
+
## Table of contents
|
|
14
|
+
|
|
15
|
+
- [Requirements](#requirements)
|
|
16
|
+
- [Installation](#installation)
|
|
17
|
+
- [Usage](#usage)
|
|
18
|
+
- [Options](#options)
|
|
19
|
+
- [Resuming after an interruption](#resuming-after-an-interruption)
|
|
20
|
+
- [Limits & duration](#limits--duration)
|
|
21
|
+
- [What to do with the WXR file](#what-to-do-with-the-wxr-file)
|
|
22
|
+
- [Programmatic API](#programmatic-api)
|
|
23
|
+
- [Security & data](#security--data)
|
|
24
|
+
- [GDPR — read before exporting](#gdpr--read-before-exporting)
|
|
25
|
+
- [Contributing](#contributing)
|
|
26
|
+
- [License](#license)
|
|
27
|
+
|
|
28
|
+
## Requirements
|
|
29
|
+
|
|
30
|
+
- **Node.js ≥ 18** (the tool uses native `fetch`, no external network dependency).
|
|
31
|
+
- A **Disqus application** ([disqus.com/api/applications/](https://disqus.com/api/applications/))
|
|
32
|
+
owned by the **forum owner**, with the **email** permission granted
|
|
33
|
+
(read-only "Read" access is enough for the export). From it you need:
|
|
34
|
+
- the **secret key** (`api_secret`) — **required**;
|
|
35
|
+
- the **access token** — **required**;
|
|
36
|
+
- the *public key* (`api_key`) — **optional** (no call made by the tool uses it).
|
|
37
|
+
|
|
38
|
+
> The **email** permission is what allows the API to return the commenters' real
|
|
39
|
+
> addresses. Without it, the export still works but emails will be missing.
|
|
40
|
+
> See [GDPR](#gdpr--read-before-exporting) before enabling this scope.
|
|
41
|
+
|
|
42
|
+
## Installation
|
|
43
|
+
|
|
44
|
+
No permanent installation is needed — the tool runs through `npx`:
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
npx disqus-wxr-middleware --help
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
For a project-local installation:
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
npm install disqus-wxr-middleware
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
## Usage
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
# Export a forum to a WXR file
|
|
60
|
+
npx disqus-wxr-middleware --forum <shortname> --out ./export.wxr.xml \
|
|
61
|
+
--api-secret <SECRET> --access-token <TOKEN>
|
|
62
|
+
|
|
63
|
+
# Credentials can also be passed through the environment
|
|
64
|
+
DISQUS_API_SECRET=<SECRET> DISQUS_ACCESS_TOKEN=<TOKEN> \
|
|
65
|
+
npx disqus-wxr-middleware --forum <shortname> --out ./export.wxr.xml
|
|
66
|
+
|
|
67
|
+
# Validate the configuration without running the export (no API call)
|
|
68
|
+
npx disqus-wxr-middleware --forum <shortname> --api-secret <SECRET> --access-token <TOKEN> --dry-run
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
The `<shortname>` is your Disqus forum's short identifier (the one in the URL
|
|
72
|
+
`https://<shortname>.disqus.com/`).
|
|
73
|
+
|
|
74
|
+
## Options
|
|
75
|
+
|
|
76
|
+
| Option | Description | Default |
|
|
77
|
+
|---|---|---|
|
|
78
|
+
| `--forum <shortname>` | Shortname of the Disqus forum to export (**required**). | — |
|
|
79
|
+
| `--out <file.xml>` | Output WXR file. | `./disqus-export.wxr.xml` |
|
|
80
|
+
| `--api-secret <secret>` | Disqus secret key (or env `DISQUS_API_SECRET`) — **required**. | — |
|
|
81
|
+
| `--access-token <token>` | Disqus access token (or env `DISQUS_ACCESS_TOKEN`) — **required**. | — |
|
|
82
|
+
| `--api-key <key>` | Disqus public key (or env `DISQUS_API_KEY`) — optional, unused. | — |
|
|
83
|
+
| `--resume` | Resumes the export from the last checkpoint (see below). | off |
|
|
84
|
+
| `--rate-budget <n>` | API request budget per hour (Disqus rate limit). | `1000` |
|
|
85
|
+
| `--max-buffer <n>` | Maximum number of comments held in memory before flushing to disk. | `50000` |
|
|
86
|
+
| `--dry-run` | Validates the configuration without running the export (no API call). | off |
|
|
87
|
+
| `-v, --version` | Prints the version and exits. | — |
|
|
88
|
+
| `--help` | Prints the full help. | — |
|
|
89
|
+
|
|
90
|
+
`npx disqus-wxr-middleware --help` lists all up-to-date options.
|
|
91
|
+
|
|
92
|
+
## Resuming after an interruption
|
|
93
|
+
|
|
94
|
+
A large forum can take **hours, or even days** to retrieve (the Disqus API rate
|
|
95
|
+
limit spreads the export over time — see [Limits & duration](#limits--duration)).
|
|
96
|
+
An interruption (network, machine shutdown, temporarily exhausted Disqus quota)
|
|
97
|
+
must not force you to start over.
|
|
98
|
+
|
|
99
|
+
Re-run **the same command** with `--resume`: the export **picks up where it
|
|
100
|
+
stopped**, without re-downloading or duplicating already-exported comments.
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
npx disqus-wxr-middleware --forum <shortname> --out ./export.wxr.xml \
|
|
104
|
+
--api-secret <SECRET> --access-token <TOKEN> --resume
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
## Limits & duration
|
|
108
|
+
|
|
109
|
+
The channel that returns emails (the Disqus API) is also the one that **scales the
|
|
110
|
+
worst**. The API is capped at **1000 requests per hour**, and each request returns
|
|
111
|
+
at most **100 comments** — a maximum throughput of **~100,000 comments per hour**.
|
|
112
|
+
|
|
113
|
+
| Forum size | Minimum export duration |
|
|
114
|
+
|---|---|
|
|
115
|
+
| 100,000 comments | ~1 h |
|
|
116
|
+
| 1,000,000 comments | ~10 h |
|
|
117
|
+
| 5,000,000 comments | ~2 days |
|
|
118
|
+
| 10,000,000 comments | ~4 days |
|
|
119
|
+
|
|
120
|
+
The tool handles the rate limit automatically (throttling, recovery after a `429`)
|
|
121
|
+
and spreads requests across hourly windows. For a very large forum, combine with
|
|
122
|
+
`--resume` to withstand interruptions.
|
|
123
|
+
|
|
124
|
+
> **History goes back ~5 years.** The Disqus API caps how far back history can be
|
|
125
|
+
> retrieved to roughly 5 years: older comments may not be exportable.
|
|
126
|
+
|
|
127
|
+
## What to do with the WXR file
|
|
128
|
+
|
|
129
|
+
The generated `.wxr.xml` file is a **standard WordPress export (WXR 1.2)**. Import
|
|
130
|
+
it into GraphComment through the **WordPress import flow** (the same one used for a
|
|
131
|
+
regular WordPress export): the Disqus threads and comments are recreated there,
|
|
132
|
+
preserving the reply hierarchy.
|
|
133
|
+
|
|
134
|
+
## Programmatic API
|
|
135
|
+
|
|
136
|
+
Besides the CLI, the package exposes a programmatic entry point so it can be driven
|
|
137
|
+
from another Node project (for example a hosted export service):
|
|
138
|
+
|
|
139
|
+
```js
|
|
140
|
+
// ESM-only package — a CommonJS project MUST use dynamic import (never `require()`).
|
|
141
|
+
const { runExport, createWxrWriter, createLogger } = await import('disqus-wxr-middleware');
|
|
142
|
+
|
|
143
|
+
const logger = createLogger();
|
|
144
|
+
const stats = await runExport({
|
|
145
|
+
config: {
|
|
146
|
+
forum: 'my-forum',
|
|
147
|
+
out: 'export.wxr.xml',
|
|
148
|
+
apiSecret: process.env.DISQUS_API_SECRET,
|
|
149
|
+
accessToken: process.env.DISQUS_ACCESS_TOKEN,
|
|
150
|
+
},
|
|
151
|
+
logger,
|
|
152
|
+
});
|
|
153
|
+
// stats → { items, comments, orphanThreads, orphanComments, flushes, out }
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
Exported functions: `runExport`, `createDisqusClient`, `createWxrWriter`,
|
|
157
|
+
`createLogger` (+ `mask`), and the checkpoint helpers (`checkpointPath`,
|
|
158
|
+
`readCheckpoint`, `writeCheckpoint`, `deleteCheckpoint`, `CheckpointError`).
|
|
159
|
+
|
|
160
|
+
- **`logger`** must expose six methods — `error`, `warn`, `info`, `debug`,
|
|
161
|
+
`progress`, `budget`. `createLogger()` returns a compliant instance; pass your own
|
|
162
|
+
only if it implements all six.
|
|
163
|
+
- **`createWriter`** can be injected into `runExport` to stream the WXR somewhere
|
|
164
|
+
other than a local file (`createWxrWriter` accepts a `Writable` in place of a
|
|
165
|
+
path).
|
|
166
|
+
|
|
167
|
+
## Security & data
|
|
168
|
+
|
|
169
|
+
- Disqus credentials are **never** written to logs or error messages (systematic
|
|
170
|
+
masking).
|
|
171
|
+
- The generated WXR file may contain **plain-text emails** (see
|
|
172
|
+
[GDPR](#gdpr--read-before-exporting)). The project's `.gitignore` already ignores
|
|
173
|
+
sensitive outputs (`*.wxr.xml`, `disqus-export*.xml`, `*.checkpoint.json`) —
|
|
174
|
+
**never commit an export or a checkpoint to a public repository**, and store them
|
|
175
|
+
on a medium you control.
|
|
176
|
+
|
|
177
|
+
## GDPR — read before exporting
|
|
178
|
+
|
|
179
|
+
> ⚠️ **The Disqus API returns commenters' emails in plain text to the forum
|
|
180
|
+
> owner**, whereas Disqus's public export omits them and the moderation interface
|
|
181
|
+
> only shows them obfuscated. Exporting these emails with this tool may therefore
|
|
182
|
+
> **bypass an anonymization measure** that Disqus otherwise applies.
|
|
183
|
+
|
|
184
|
+
This tool **observes** this API behavior; it takes **no position** on its
|
|
185
|
+
lawfulness. As the **data controller** of your forum, it is your responsibility to
|
|
186
|
+
verify:
|
|
187
|
+
|
|
188
|
+
- the **legal basis** for retrieving and storing these emails;
|
|
189
|
+
- the **information given to the commenters** whose data you retrieve;
|
|
190
|
+
- the appropriate **retention period**.
|
|
191
|
+
|
|
192
|
+
These obligations fall on the **client** running the tool, not on GraphComment. If
|
|
193
|
+
you are migrating to GraphComment, refer to the legal guidance provided by your
|
|
194
|
+
GraphComment contacts regarding email retention on the platform side.
|
|
195
|
+
|
|
196
|
+
## Contributing
|
|
197
|
+
|
|
198
|
+
Contributions are welcome — see [CONTRIBUTING.md](./CONTRIBUTING.md).
|
|
199
|
+
|
|
200
|
+
## License
|
|
201
|
+
|
|
202
|
+
[MIT](./LICENSE) © 2026 Semiologic
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// CLI entry point of the Disqus → WXR middleware.
|
|
3
|
+
// All the logic lives in src/cli.js (testable without spawning a process). Here: shebang +
|
|
4
|
+
// delegation + catching unexpected errors (expected errors — missing credentials —
|
|
5
|
+
// set process.exitCode themselves and never bubble up this far).
|
|
6
|
+
import { run } from '../src/cli.js';
|
|
7
|
+
|
|
8
|
+
run().catch((err) => {
|
|
9
|
+
// Safety net: never let a raw stack trace leak a secret.
|
|
10
|
+
process.stderr.write(`[error] Unexpected error: ${err?.message ?? err}\n`);
|
|
11
|
+
process.exitCode = 1;
|
|
12
|
+
});
|
package/package.json
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "disqus-wxr-middleware",
|
|
3
|
+
"version": "0.3.0",
|
|
4
|
+
"description": "Exports a Disqus forum to a WordPress WXR file through the Disqus API (client-run tool, no WordPress required).",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"bin": {
|
|
7
|
+
"disqus-wxr": "bin/disqus-wxr.js"
|
|
8
|
+
},
|
|
9
|
+
"main": "src/index.js",
|
|
10
|
+
"exports": {
|
|
11
|
+
".": "./src/index.js"
|
|
12
|
+
},
|
|
13
|
+
"engines": {
|
|
14
|
+
"node": ">=18"
|
|
15
|
+
},
|
|
16
|
+
"scripts": {
|
|
17
|
+
"test": "node --test"
|
|
18
|
+
},
|
|
19
|
+
"files": [
|
|
20
|
+
"bin/",
|
|
21
|
+
"src/",
|
|
22
|
+
"LICENSE",
|
|
23
|
+
"README.md"
|
|
24
|
+
],
|
|
25
|
+
"keywords": [
|
|
26
|
+
"disqus",
|
|
27
|
+
"wxr",
|
|
28
|
+
"wordpress",
|
|
29
|
+
"export",
|
|
30
|
+
"migration",
|
|
31
|
+
"comments"
|
|
32
|
+
],
|
|
33
|
+
"license": "MIT",
|
|
34
|
+
"author": "Semiologic",
|
|
35
|
+
"repository": {
|
|
36
|
+
"type": "git",
|
|
37
|
+
"url": "git+https://github.com/graphcomment/disqus-wxr-middleware.git"
|
|
38
|
+
},
|
|
39
|
+
"bugs": {
|
|
40
|
+
"url": "https://github.com/graphcomment/disqus-wxr-middleware/issues"
|
|
41
|
+
},
|
|
42
|
+
"homepage": "https://github.com/graphcomment/disqus-wxr-middleware#readme",
|
|
43
|
+
"dependencies": {
|
|
44
|
+
"commander": "^12.1.0"
|
|
45
|
+
},
|
|
46
|
+
"devDependencies": {
|
|
47
|
+
"sax": "^1.6.0"
|
|
48
|
+
}
|
|
49
|
+
}
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
// Persistence of the resume checkpoint (B2_01) + the injectable checkpoint STORE (B4_03, C11).
|
|
2
|
+
//
|
|
3
|
+
// TWO layers live here:
|
|
4
|
+
// 1. The historical low-level FS helpers (`checkpointPath`/`readCheckpoint`/`writeCheckpoint`/
|
|
5
|
+
// `deleteCheckpoint`/`CheckpointError`) — a local JSON file next to `--out`. UNCHANGED,
|
|
6
|
+
// still exported (C10, tested by test/api.test.js). They back the default store below.
|
|
7
|
+
// 2. `createFsCheckpointStore(outPath)` (B4_03) — the DEFAULT implementation of the injectable
|
|
8
|
+
// `checkpointStore` the resumable pipeline consumes. It owns BOTH the checkpoint I/O AND the
|
|
9
|
+
// physical WXR stream lifecycle (fresh open / resume rewind / durable-offset measurement), so
|
|
10
|
+
// `runResumableExport` no longer contains any `fs.*` of its own. A server (api/, B5_02) injects
|
|
11
|
+
// its OWN store (checkpoint on the Mongo job, stream to GridFS) implementing the same shape.
|
|
12
|
+
//
|
|
13
|
+
// The local JSON checkpoint materializes the durable state of the export so `--resume` picks up
|
|
14
|
+
// without starting over:
|
|
15
|
+
// { forum, cursor, exhausted, comments_written, bytes_written, threads_resolved,
|
|
16
|
+
// started_at, last_page_at }
|
|
17
|
+
// - `cursor` : Disqus boundary cursor of the LAST fully flushed page
|
|
18
|
+
// (pagination resume point); null = beginning / no more pages.
|
|
19
|
+
// - `exhausted` : true = Disqus stream exhausted (no more pages) → distinguishes
|
|
20
|
+
// "cursor null = end" from "cursor null = beginning" on resume.
|
|
21
|
+
// - `bytes_written`: DURABLE byte offset of the WXR file = truncation target on resume.
|
|
22
|
+
// Invariant: bytes [0, bytes_written) contain exactly the comments
|
|
23
|
+
// of the pages strictly before `cursor`. (FS store only — an injected store
|
|
24
|
+
// may treat this field as an opaque durable position of its own.)
|
|
25
|
+
//
|
|
26
|
+
// ⚠ NO Disqus credential (`api_secret`/`access_token`/`api_key`) EVER appears in the
|
|
27
|
+
// checkpoint — the file stays on the client's machine. The shape is closed (the pipeline
|
|
28
|
+
// only writes the 8 keys above) and a test verifies it explicitly. An injected store MUST
|
|
29
|
+
// keep the same opacity: the checkpoint object it persists is the pipeline's, never enriched
|
|
30
|
+
// with a token.
|
|
31
|
+
//
|
|
32
|
+
// ATOMIC write (tmp + fsync + rename): a crash during the write can never leave a
|
|
33
|
+
// half-written checkpoint — the rename only publishes the complete file.
|
|
34
|
+
|
|
35
|
+
import * as fs from 'node:fs';
|
|
36
|
+
import { createWriteStream } from 'node:fs';
|
|
37
|
+
|
|
38
|
+
// Canonical checkpoint path derived from the WXR output file.
|
|
39
|
+
export function checkpointPath(out) {
|
|
40
|
+
return `${out}.checkpoint.json`;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// Error: checkpoint present but unreadable (corrupted JSON / I/O). Distinct from "absent"
|
|
44
|
+
// (readCheckpoint returns null when the file simply does not exist). Carries an exitCode
|
|
45
|
+
// for the entry point. Never contains a credential.
|
|
46
|
+
export class CheckpointError extends Error {
|
|
47
|
+
constructor(message) {
|
|
48
|
+
super(message);
|
|
49
|
+
this.name = 'CheckpointError';
|
|
50
|
+
this.exitCode = 1;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// Reads the checkpoint. Returns the parsed object, or `null` if the file does not exist
|
|
55
|
+
// (normal case of a first run). Throws `CheckpointError` if the file exists but is
|
|
56
|
+
// unreadable or is not valid JSON (the pipeline turns that into a clear message).
|
|
57
|
+
export function readCheckpoint(path) {
|
|
58
|
+
let raw;
|
|
59
|
+
try {
|
|
60
|
+
raw = fs.readFileSync(path, 'utf8');
|
|
61
|
+
} catch (err) {
|
|
62
|
+
if (err.code === 'ENOENT') return null;
|
|
63
|
+
throw new CheckpointError(`unreadable checkpoint (${path}): ${err.message}`);
|
|
64
|
+
}
|
|
65
|
+
try {
|
|
66
|
+
return JSON.parse(raw);
|
|
67
|
+
} catch (err) {
|
|
68
|
+
throw new CheckpointError(`corrupted checkpoint (${path}): ${err.message}`);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// Writes the checkpoint ATOMICALLY: serializes into `<path>.tmp`, fsync, then rename
|
|
73
|
+
// onto `path` (atomic within the same filesystem). The current checkpoint is therefore
|
|
74
|
+
// always either the old complete one or the new complete one — never a mix.
|
|
75
|
+
export function writeCheckpoint(path, checkpoint) {
|
|
76
|
+
const tmp = `${path}.tmp`;
|
|
77
|
+
const fd = fs.openSync(tmp, 'w');
|
|
78
|
+
try {
|
|
79
|
+
fs.writeFileSync(fd, JSON.stringify(checkpoint, null, 2));
|
|
80
|
+
fs.fsyncSync(fd);
|
|
81
|
+
} finally {
|
|
82
|
+
fs.closeSync(fd);
|
|
83
|
+
}
|
|
84
|
+
fs.renameSync(tmp, path);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// Deletes the checkpoint if it exists: successful end of run (nothing left to resume) OR
|
|
88
|
+
// purge of a stale checkpoint at the start of a fresh run (no `--resume`). ENOENT ignored.
|
|
89
|
+
export function deleteCheckpoint(path) {
|
|
90
|
+
try {
|
|
91
|
+
fs.unlinkSync(path);
|
|
92
|
+
} catch (err) {
|
|
93
|
+
if (err.code !== 'ENOENT') throw err;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// ─── Injectable checkpoint STORE (B4_03, contract C11) ──────────────────────────────────────
|
|
98
|
+
//
|
|
99
|
+
// createFsCheckpointStore(outPath) → the DEFAULT store the resumable pipeline uses. It is the
|
|
100
|
+
// existing FS behavior, moved BEHIND an interface — byte-for-byte identical (test/resume.test.js
|
|
101
|
+
// stays green). A consumer injects a store of the SAME SHAPE to resume against a non-seekable
|
|
102
|
+
// target (GridFS) and a checkpoint persisted off the filesystem.
|
|
103
|
+
//
|
|
104
|
+
// Store contract (all async — the injected implementations do real I/O):
|
|
105
|
+
// read() → Promise<checkpoint|null> persisted checkpoint, or null if none.
|
|
106
|
+
// Throws (CheckpointError-like) if present but unreadable.
|
|
107
|
+
// write(checkpoint) → Promise<void> persists atomically.
|
|
108
|
+
// delete() → Promise<void> removes the checkpoint (idempotent).
|
|
109
|
+
// open() → Promise<Writable> FRESH run: a Writable to write the WXR
|
|
110
|
+
// from the start (FS: truncate/create in 'w' mode).
|
|
111
|
+
// rewindTo(checkpoint) → Promise<Writable> RESUME: rewinds the target to the durable
|
|
112
|
+
// boundary of `checkpoint` and returns a Writable positioned to append
|
|
113
|
+
// from there (FS: truncate the file to `bytes_written`, reopen in 'a').
|
|
114
|
+
// Throws if the target is inconsistent with the checkpoint (missing /
|
|
115
|
+
// shorter than the durable offset) — never resumes onto a bad target.
|
|
116
|
+
// durableOffset(stream)→ Promise<number> the durable position of everything written
|
|
117
|
+
// to `stream` so far (FS: write-barrier + fsync + fstat on the fd). The
|
|
118
|
+
// pipeline stores this as the checkpoint's `bytes_written`. An injected
|
|
119
|
+
// store returns whatever opaque durable position it can guarantee.
|
|
120
|
+
//
|
|
121
|
+
// The pipeline treats `bytes_written`/the offset as OPAQUE: it reads it from durableOffset() and
|
|
122
|
+
// hands it back via the same store's rewindTo() — it never interprets it as bytes itself.
|
|
123
|
+
export function createFsCheckpointStore(outPath) {
|
|
124
|
+
const path = checkpointPath(outPath);
|
|
125
|
+
|
|
126
|
+
return {
|
|
127
|
+
async read() {
|
|
128
|
+
return readCheckpoint(path); // throws CheckpointError if corrupted; null if absent
|
|
129
|
+
},
|
|
130
|
+
|
|
131
|
+
async write(checkpoint) {
|
|
132
|
+
writeCheckpoint(path, checkpoint);
|
|
133
|
+
},
|
|
134
|
+
|
|
135
|
+
async delete() {
|
|
136
|
+
deleteCheckpoint(path);
|
|
137
|
+
},
|
|
138
|
+
|
|
139
|
+
// Fresh run: (re)create the WXR file empty and return an append/truncate write stream.
|
|
140
|
+
// 'w' truncates any pre-existing file — matching the previous fresh-run behavior.
|
|
141
|
+
async open() {
|
|
142
|
+
return createWriteStream(outPath, { flags: 'w', encoding: 'utf8' });
|
|
143
|
+
},
|
|
144
|
+
|
|
145
|
+
// Resume: validate the file against the checkpoint, truncate it to the durable offset
|
|
146
|
+
// (strips the footer + any partial post-checkpoint write), then reopen in append mode.
|
|
147
|
+
// This is exactly the sequence the pipeline used to run inline (fs.statSync + fs.truncateSync
|
|
148
|
+
// + createWriteStream {flags:'a'}), moved here unchanged so resume.test.js keeps passing.
|
|
149
|
+
async rewindTo(checkpoint) {
|
|
150
|
+
let size;
|
|
151
|
+
try {
|
|
152
|
+
size = fs.statSync(outPath).size;
|
|
153
|
+
} catch {
|
|
154
|
+
throw new Error(
|
|
155
|
+
`Checkpoint present but output file ${outPath} not found — cannot resume. ` +
|
|
156
|
+
`Delete the checkpoint to start over.`,
|
|
157
|
+
);
|
|
158
|
+
}
|
|
159
|
+
if (size < checkpoint.bytes_written) {
|
|
160
|
+
throw new Error(
|
|
161
|
+
`Output file ${outPath} (${size} B) shorter than the checkpoint ` +
|
|
162
|
+
`(${checkpoint.bytes_written} B) — inconsistent state. Delete the checkpoint to start over.`,
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
fs.truncateSync(outPath, checkpoint.bytes_written); // strips footer + partial post-checkpoint data
|
|
166
|
+
return createWriteStream(outPath, { flags: 'a', encoding: 'utf8' });
|
|
167
|
+
},
|
|
168
|
+
|
|
169
|
+
// DURABLE byte offset of the file, read AFTER everything the writer wrote has actually
|
|
170
|
+
// reached the fd: write barrier (callback of an empty write → all prior writes, FIFO
|
|
171
|
+
// ordered, are processed), then fsync (OS → disk), then fstat. Without this barrier,
|
|
172
|
+
// the offset could undercount (chunks still in Node's buffer) → on resume, a too-short
|
|
173
|
+
// truncation would lose the tail of an already-written page.
|
|
174
|
+
async durableOffset(stream) {
|
|
175
|
+
await new Promise((resolve, reject) => {
|
|
176
|
+
stream.write(Buffer.alloc(0), (err) => (err ? reject(err) : resolve()));
|
|
177
|
+
});
|
|
178
|
+
fs.fsyncSync(stream.fd);
|
|
179
|
+
return fs.fstatSync(stream.fd).size;
|
|
180
|
+
},
|
|
181
|
+
};
|
|
182
|
+
}
|
package/src/cli.js
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
// CLI of the Disqus → WXR middleware.
|
|
2
|
+
// Responsibility (B0_01): declare the options, handle --help/--version, resolve and
|
|
3
|
+
// validate the configuration (credentials included), then stop. No API call and no
|
|
4
|
+
// WXR generation here — B1_01→B1_03 plug onto the validated config + the logger.
|
|
5
|
+
import { createRequire } from 'node:module';
|
|
6
|
+
import { Command } from 'commander';
|
|
7
|
+
import { resolveConfig, MissingCredentialsError } from './config.js';
|
|
8
|
+
import { createLogger, mask } from './logger.js';
|
|
9
|
+
import { runExport, DEFAULT_MAX_BUFFER } from './pipeline.js';
|
|
10
|
+
|
|
11
|
+
const require = createRequire(import.meta.url);
|
|
12
|
+
const pkg = require('../package.json');
|
|
13
|
+
|
|
14
|
+
// Builds the commander program. Exported for tests / introspection.
|
|
15
|
+
export function buildProgram() {
|
|
16
|
+
const program = new Command();
|
|
17
|
+
program
|
|
18
|
+
.name('disqus-wxr')
|
|
19
|
+
.description(
|
|
20
|
+
'Exports a Disqus forum to a WordPress WXR file through the Disqus API.\n' +
|
|
21
|
+
'Standalone tool, run by the client with their own Disqus credentials (outside WordPress).',
|
|
22
|
+
)
|
|
23
|
+
// --version and --help are handled natively by commander and short-circuit
|
|
24
|
+
// any validation (including the required --forum), exiting with code 0.
|
|
25
|
+
.version(pkg.version, '-v, --version', 'print the version and exit')
|
|
26
|
+
.requiredOption('--forum <shortname>', 'shortname of the Disqus forum to export (required)')
|
|
27
|
+
.option('--out <file.xml>', 'output WXR file', './disqus-export.wxr.xml')
|
|
28
|
+
.option(
|
|
29
|
+
'--api-key <key>',
|
|
30
|
+
'Disqus public key (or env DISQUS_API_KEY) — optional: no call uses it',
|
|
31
|
+
)
|
|
32
|
+
.option('--api-secret <secret>', 'Disqus secret key (or env DISQUS_API_SECRET) — required')
|
|
33
|
+
.option('--access-token <token>', 'Disqus access token (or env DISQUS_ACCESS_TOKEN) — required')
|
|
34
|
+
.option('--resume', 'resume from the last checkpoint', false)
|
|
35
|
+
.option(
|
|
36
|
+
'--rate-budget <n>',
|
|
37
|
+
'API request budget per hour (Disqus rate limit)',
|
|
38
|
+
'1000',
|
|
39
|
+
)
|
|
40
|
+
.option(
|
|
41
|
+
'--max-buffer <n>',
|
|
42
|
+
'maximum number of buffered comments before flushing (per-thread grouping over the global stream)',
|
|
43
|
+
String(DEFAULT_MAX_BUFFER),
|
|
44
|
+
)
|
|
45
|
+
.option('--dry-run', 'validate the configuration without running the export (no API call)', false);
|
|
46
|
+
return program;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// Logical entry point. Returns the validated config (reused by B1+) or null
|
|
50
|
+
// if validation failed (process.exitCode is then set to 1).
|
|
51
|
+
export async function run(argv = process.argv) {
|
|
52
|
+
const program = buildProgram();
|
|
53
|
+
program.parse(argv); // commander handles --help / --version here (exit 0)
|
|
54
|
+
|
|
55
|
+
const logger = createLogger({ level: process.env.DISQUS_WXR_LOG_LEVEL || 'info' });
|
|
56
|
+
const opts = program.opts();
|
|
57
|
+
|
|
58
|
+
let config;
|
|
59
|
+
try {
|
|
60
|
+
config = resolveConfig(opts, process.env);
|
|
61
|
+
} catch (err) {
|
|
62
|
+
if (err instanceof MissingCredentialsError) {
|
|
63
|
+
logger.error(err.message);
|
|
64
|
+
process.exitCode = err.exitCode; // 1
|
|
65
|
+
return null;
|
|
66
|
+
}
|
|
67
|
+
throw err;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// Config ready.
|
|
71
|
+
logger.info(`Configuration validated for forum "${config.forum}".`);
|
|
72
|
+
// Credential details are only emitted at debug level, and ALWAYS masked.
|
|
73
|
+
logger.debug(
|
|
74
|
+
`Credentials — secret ${mask(config.apiSecret)}, token ${mask(config.accessToken)}, ` +
|
|
75
|
+
`api-key ${mask(config.apiKey)}.`,
|
|
76
|
+
);
|
|
77
|
+
logger.info(
|
|
78
|
+
`WXR output: ${config.out} · API budget: ${config.rateBudget} req/h · ` +
|
|
79
|
+
`resume: ${config.resume ? 'yes' : 'no'}.`,
|
|
80
|
+
);
|
|
81
|
+
|
|
82
|
+
// --max-buffer: maximum number of buffered comments before flushing (per-thread grouping
|
|
83
|
+
// over the global ascending stream — local decision "bounded configurable" 2026-06-30).
|
|
84
|
+
// Strictly positive integer, otherwise default (same validation as --rate-budget). Read
|
|
85
|
+
// HERE, not in resolveConfig (whose output contract for B1_01 is frozen/tested), and
|
|
86
|
+
// passed directly to runExport.
|
|
87
|
+
const rawMaxBuffer = String(opts.maxBuffer ?? '').trim();
|
|
88
|
+
const maxBuffer =
|
|
89
|
+
/^\d+$/.test(rawMaxBuffer) && Number.parseInt(rawMaxBuffer, 10) > 0
|
|
90
|
+
? Number.parseInt(rawMaxBuffer, 10)
|
|
91
|
+
: DEFAULT_MAX_BUFFER;
|
|
92
|
+
|
|
93
|
+
// --dry-run: validate the config without running the export (check the credentials
|
|
94
|
+
// without spending API budget). Short-circuits BEFORE any network call.
|
|
95
|
+
if (opts.dryRun) {
|
|
96
|
+
logger.info(`--dry-run mode: configuration OK (max buffer ${maxBuffer}), export not started.`);
|
|
97
|
+
return config;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// Actual wiring (B1_03): Disqus export → streaming WXR. A business error (Disqus
|
|
101
|
+
// auth/quota, file I/O) is turned into a clear message + exit 1 (never a raw stack
|
|
102
|
+
// trace that could leak a secret; the safety net in bin/ then sees nothing bubble up).
|
|
103
|
+
try {
|
|
104
|
+
return await runExport({ config, logger, maxBuffer });
|
|
105
|
+
} catch (err) {
|
|
106
|
+
logger.error(`Export failed: ${err?.message ?? err}`);
|
|
107
|
+
process.exitCode = err?.exitCode ?? 1;
|
|
108
|
+
return null;
|
|
109
|
+
}
|
|
110
|
+
}
|
package/src/config.js
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
// Resolution + validation of the middleware configuration.
|
|
2
|
+
// Credential sources: CLI flags ∪ environment variables, flags taking precedence.
|
|
3
|
+
// REQUIRED set = api_secret + access_token (D2, decided 2026-06-30): the middleware's
|
|
4
|
+
// single API call (`posts/list.json?forum=…&api_secret=…&access_token=…`, audit §5.A)
|
|
5
|
+
// does NOT use the public key — it is accepted but never required.
|
|
6
|
+
|
|
7
|
+
// Business error: one or more required credentials are missing. Carries its own
|
|
8
|
+
// exit code so the entry point does not have to guess it. The message NEVER contains
|
|
9
|
+
// a credential value (only the name of what is missing).
|
|
10
|
+
export class MissingCredentialsError extends Error {
|
|
11
|
+
constructor(missing) {
|
|
12
|
+
super(
|
|
13
|
+
`Missing Disqus credentials: ${missing.join(', ')}.\n` +
|
|
14
|
+
'Create an app at https://disqus.com/api/applications/ and grant it the "email" ' +
|
|
15
|
+
'permission (read-only "Read" access is enough for the export), then provide the ' +
|
|
16
|
+
'secret key and the access token through --api-secret / --access-token or the ' +
|
|
17
|
+
'DISQUS_API_SECRET / DISQUS_ACCESS_TOKEN environment variables.',
|
|
18
|
+
);
|
|
19
|
+
this.name = 'MissingCredentialsError';
|
|
20
|
+
this.exitCode = 1;
|
|
21
|
+
this.missing = missing;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
// Returns the first non-empty candidate (flag then env), TRIMMED, otherwise undefined.
|
|
26
|
+
// - treats `DISQUS_API_SECRET=""` or `" "` as absent (emptied-env trap in tests/CI);
|
|
27
|
+
// - trims the retained value (a credential pasted with stray whitespace would
|
|
28
|
+
// otherwise break the Disqus URL in B1 → 401).
|
|
29
|
+
function firstNonEmpty(value, fallback) {
|
|
30
|
+
for (const candidate of [value, fallback]) {
|
|
31
|
+
if (candidate !== undefined && candidate !== null && String(candidate).trim() !== '') {
|
|
32
|
+
return String(candidate).trim();
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
return undefined;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// Resolves the config from the commander options and the environment.
|
|
39
|
+
// Throws MissingCredentialsError if the required set is not complete.
|
|
40
|
+
export function resolveConfig(opts, env = process.env) {
|
|
41
|
+
const apiKey = firstNonEmpty(opts.apiKey, env.DISQUS_API_KEY);
|
|
42
|
+
const apiSecret = firstNonEmpty(opts.apiSecret, env.DISQUS_API_SECRET);
|
|
43
|
+
const accessToken = firstNonEmpty(opts.accessToken, env.DISQUS_ACCESS_TOKEN);
|
|
44
|
+
|
|
45
|
+
const missing = [];
|
|
46
|
+
if (!apiSecret) missing.push('--api-secret (or DISQUS_API_SECRET)');
|
|
47
|
+
if (!accessToken) missing.push('--access-token (or DISQUS_ACCESS_TOKEN)');
|
|
48
|
+
if (missing.length > 0) throw new MissingCredentialsError(missing);
|
|
49
|
+
|
|
50
|
+
// --rate-budget: STRICTLY positive integer (digits only), otherwise fall back to the
|
|
51
|
+
// default 1000 (measured Disqus rate limit, audit §1.B). Strict validation rather than
|
|
52
|
+
// `parseInt`, which would silently truncate `2.7`→2 (a typo would throttle the export
|
|
53
|
+
// in B1). The actual wiring — and the user warning — arrive in B1_01.
|
|
54
|
+
const rawBudget = String(opts.rateBudget ?? '').trim();
|
|
55
|
+
const rateBudget =
|
|
56
|
+
/^\d+$/.test(rawBudget) && Number.parseInt(rawBudget, 10) > 0
|
|
57
|
+
? Number.parseInt(rawBudget, 10)
|
|
58
|
+
: 1000;
|
|
59
|
+
|
|
60
|
+
return {
|
|
61
|
+
forum: opts.forum,
|
|
62
|
+
out: opts.out,
|
|
63
|
+
apiKey: apiKey ?? null,
|
|
64
|
+
apiSecret,
|
|
65
|
+
accessToken,
|
|
66
|
+
resume: Boolean(opts.resume),
|
|
67
|
+
rateBudget,
|
|
68
|
+
};
|
|
69
|
+
}
|