imapkit 4.1.1 → 4.2.0

package/README.md

@@ -65,7 +65,7 @@ ImapKit is a single user, multiple connection IMAP server. Changes made over IMA
 
 Several clients can connect to the server simultaneously but all the clients share the same user account, even if login credentials are different. The ACL plugin can limit what users other than the owner can do (see [ACL](#acl)).
 
-ImapKit is extendable: any command can be overridden and plugins can be added (see [Creating custom plugins](#creating-custom-plugins), and `src/commands` and `src/plugins` for the built-in commands and plugins).
+ImapKit is extendable: any command can be overridden and plugins can be added (see [Creating custom plugins](#creating-custom-plugins), and `src/commands` and `src/plugins` for the built-in commands and plugins). To test a client against a broken or unusual server, [script rules](#scripted-faults) make the server misbehave at chosen points.
 
 ## Strict by design
 
@@ -395,6 +395,92 @@ describe('IMAP tests', () => {
 });
 ```
 
+## Scripted faults
+
+ImapKit is strict and correct by default. To test how a client copes with a server that is not, script rules make the server deviate from the protocol at chosen points: answer a command with a canned response, send a literal where a quoted string is expected, cut a response in the middle of a literal, delay or split output, or drop the connection. Rules come from the `script` server option (a rule or a list of rules), or are added at runtime with `server.script.add()`:
+
+```javascript
+const server = imapkit({
+    plugins: ['IDLE'],
+    script: [
+        // the first SELECT gets NO, the next ones run as usual
+        { on: 'command', command: 'SELECT', times: 1, send: '$TAG NO [UNAVAILABLE] Try again later\r\n' },
+        // the body of message 1 is cut short and the connection dropped
+        { on: 'response', command: 'FETCH', match: /^\* 1 FETCH .*BODY\[\]/, truncate: 40 }
+    ]
+});
+
+// a rule added later, it returns a handle
+const rule = server.script.add({
+    on: 'response',
+    command: 'FETCH',
+    untagged: true,
+    // strings in the response tree become literals, valid IMAP that a client must handle
+    mutate: response => {
+        const walk = list =>
+            list.forEach((node, i) => (Array.isArray(node) ? walk(node) : typeof node === 'string' && (list[i] = { type: 'LITERAL', value: node })));
+        walk(response.attributes);
+    }
+});
+// ... run the client
+assert.strictEqual(rule.hits, 1);
+rule.remove();
+```
+
+Every rule watches one event (`on`):
+
+- `greeting`: the `* OK` greeting of a new connection
+- `command`: a complete command line (with its literals) from the client. The rule acts instead of the parser and the command handler, so it also matches lines that do not parse and commands that do not exist. The rule is chosen when the line arrives (matchers like `state` see the session at that moment), and acts in the order of the commands, so pipelined responses stay in order
+- `input`: a line read by a command that takes over the input, like `DONE` of IDLE or a SASL response of AUTHENTICATE
+- `response`: every response the server sends with `connection.send()`, tagged and untagged, as the exact bytes that are about to go out, after every plugin and the core changed the response
+- `continuation`: a `+` continuation request (literals, IDLE, AUTHENTICATE)
+
+The matchers of a rule all have to match. Rules are checked in the order they were added, the first rule that matches and is not used up handles the event, so a later rule can handle what an earlier one leaves alone.
+
+| Matcher                    | Events                 | Matches                                                                                                                                                            |
+| -------------------------- | ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `command`                  | all but greeting       | the command name, or a list of names, case-insensitive (`'UID FETCH'`). An unsolicited response belongs to the command that runs, or that reads input (IDLE)       |
+| `tag`                      | all but greeting       | the command tag, a string or a RegExp                                                                                                                              |
+| `description`              | response, continuation | the description passed to `connection.send()`, or a list of them. Continuation requests are `LITERAL`, `IDLE`, `AUTHENTICATE PLAIN` and `AUTHENTICATE OAUTHBEARER` |
+| `untagged`                 | response               | `true` for untagged responses only, `false` for tagged ones                                                                                                        |
+| `session`                  | all                    | the number of the connection, or a list of numbers, 1 for the first connection the server accepted                                                                 |
+| `state`, `user`, `mailbox` | all                    | the session state (`'Not Authenticated'`, `'Authenticated'`, `'Selected'`), the authenticated user, the path of the selected mailbox                               |
+| `match`                    | all                    | a RegExp, or a string with a regular expression, tested against the command line or the output bytes (a binary string)                                             |
+| `when`                     | all                    | a function that gets the event context and returns true to match                                                                                                   |
+| `nth`                      | all                    | the rule fires from the nth matching event on (default 1)                                                                                                          |
+| `times`                    | all                    | the rule fires this many times at most, then lets later rules handle the event                                                                                     |
+
+The actions say what happens instead of the usual behavior. Strings are sent as they are (binary strings, one character per octet, or UTF-8 when they have characters above U+00FF) without an added CRLF, and `$TAG` in a string is replaced with the tag of the command. A Buffer is sent as it is, a function gets the event context and returns a string or a Buffer.
+
+| Action                | Events                 | Effect                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
+| --------------------- | ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `send`                | all                    | output events: bytes sent instead of the output. command and input: bytes sent instead of processing the line                                                                                                                                                                                                                                                                                                                                                  |
+| `run`                 | command, input         | process the line as usual after `send` (to add output before the real response)                                                                                                                                                                                                                                                                                                                                                                                |
+| `drop`                | all                    | output events: send nothing. command and input: ignore the line, the client gets no answer                                                                                                                                                                                                                                                                                                                                                                     |
+| `mutate`              | response, continuation | `(response, context)` gets a copy of the response object before it is compiled and changes it, or returns another one. Continuation requests have a response object only when a plugin sends them with `connection.send()` (the error challenges of XOAUTH2 and OAUTHBEARER), other events ignore `mutate`. A tagged response that does not compile then is sent as `NO [SERVERBUG]`, an untagged one is dropped. Use `send` for output that is not valid IMAP |
+| `before`, `after`     | output events          | bytes sent before or after the output, e.g. an unsolicited response                                                                                                                                                                                                                                                                                                                                                                                            |
+| `delay`               | all but input          | milliseconds to wait before the output goes out, all later output waits behind it. For a command, the wait before the rule acts or the command runs, later commands wait too                                                                                                                                                                                                                                                                                   |
+| `chunk`, `chunkDelay` | all                    | write the bytes in pieces of `chunk` octets, `chunkDelay` milliseconds apart (default 10)                                                                                                                                                                                                                                                                                                                                                                      |
+| `truncate`            | all                    | send only this many octets of the bytes, then close the connection                                                                                                                                                                                                                                                                                                                                                                                             |
+| `close`               | all                    | close the connection after the bytes are sent, `'reset'` destroys the socket instead (a TCP RST where the runtime supports it)                                                                                                                                                                                                                                                                                                                                 |
+
+A rule needs at least one action, and for command and input events `chunk` and `truncate` need `send`. A rule with only `delay` (and `run`) delays the line and then processes it as usual. Rules are checked when they are added: an unknown option, an option that does not apply to the event, or an invalid value throws a `TypeError`, so a typo can not turn into a rule that never fires.
+
+The event context, which `when`, `mutate` and `send` functions get, has `event`, `connection`, `session`, `state`, `user`, `mailbox`, `tag`, `command`, `data` (the command line or the output bytes, as a binary string), and for responses `description` and `response`.
+
+`server.script.add(rule)` returns a handle with `rule`, `matched` (events that matched the rule, also before `nth`), `hits` (events the rule handled) and `remove()`, `server.script.add([rules])` returns a list of handles. `server.script.rules` lists the handles in order, `server.script.clear()` removes every rule. The server emits a `script` event `{ rule, event, session, tag, command }` every time a rule fires.
+
+The `imapkit` command takes the rules as JSON with `--script=<path>` (or `IMAPKIT_SCRIPT`), or as `script` in the `--config` file. JSON rules use strings for `match` and `send`, functions (`when`, `mutate`, function values of `send`) work only from JavaScript:
+
+```json
+[
+    { "on": "greeting", "send": "* BYE Too many connections\r\n", "close": true, "times": 1 },
+    { "on": "response", "command": "FETCH", "untagged": true, "send": "* 1 FETCH (BODY[] {100}\r\nshort", "close": true }
+]
+```
+
+Faults change only the output and the handling of the lines a rule matches, the state of the server stays consistent: a LOGIN answered by a rule with `OK` does not log the session in, and a dropped EXPUNGE response still removes the message. COMPRESS works with delayed output, as the output keeps the compression layer it was sent with. Script rules are for tests only, a rule can send anything.
+
 ## Creating custom plugins
 
 A plugin can be a string as a pointer to a built in plugin or a function. Plugin function is run when the server is created and gets server instance object as an argument.
@@ -589,6 +675,8 @@ server.resetHandlers.push(function (connection) {
 
 Any response sent to the client can be overridden or cancelled by other handlers. You should append your handler to `server.outputHandlers` array. If something is being sent to the client, the response object is passed through all handlers in this array.
 
+Output handlers are meant for extensions that change valid responses. They run before the core finishes the response (response codes like `EXPUNGEISSUED`, mailbox names, the 7-bit status text) and before the compiler, which refuses output that is not valid IMAP. To make the server send something wrong on purpose, use [script rules](#scripted-faults) instead, they see the final bytes.
+
     server.outputHandlers.push(function(connection, /* arguments from connection.send */){})
 
 `response` arguments from `connection.send` is an object and thus any modifications will be passed on. If `skipResponse` property is added to the response object, the data is not sent to the client.

package/bin/help.txt

@@ -16,6 +16,9 @@ Usage: imapkit [OPTS]
   -d, --debug=true         Writes IMAP traffic to console
   --storage=<path>         Path to JSON file with the directory tree
   --config=<config>        Path to config JSON file
+  --script=<path>          Path to a JSON file with script rules that
+                           make the server misbehave on purpose (see
+                           "Scripted faults" below)
   --plugin=<plugin>        Enables a plugin for the server. See below for
                            available plugins.
   --smtpPort=<port>        Port number for incoming SMTP server. If not set
@@ -23,7 +26,7 @@ Usage: imapkit [OPTS]
 
 The options can also be set with environment variables: IMAPKIT_PORT,
 IMAPKIT_SECURE, IMAPKIT_DEBUG, IMAPKIT_STORAGE, IMAPKIT_CONFIG,
-IMAPKIT_PLUGINS (comma separated) and IMAPKIT_SMTPPORT.
+IMAPKIT_SCRIPT, IMAPKIT_PLUGINS (comma separated) and IMAPKIT_SMTPPORT.
 
 NB! If port or smtpPort values are below 1024 you most probably need to
 use sudo or run the command with administrator rights.
@@ -61,6 +64,29 @@ Configuration file takes the following structure
     }
 }
 
+Scripted faults
+---------------
+
+A script file holds a list of rules. Each rule watches one event
+("greeting", "command", "input", "response" or "continuation") and
+changes what the server does when its matchers fit. For example, this
+script answers the first FETCH with a literal that is too short and
+then drops the connection:
+
+[
+    {
+        "on": "response",
+        "command": "FETCH",
+        "untagged": true,
+        "times": 1,
+        "send": "* 1 FETCH (BODY[] {100}\r\nshort",
+        "close": true
+    }
+]
+
+Strings are sent as they are, "$TAG" stands for the command tag and
+"match" is a regular expression. See the README for every option.
+
 Storage file is a tree like structure starting with namespace values.
 INBOX has its own namespace.
 

package/bin/imapkit.js

@@ -19,6 +19,7 @@ const { values: argv } = parseArgs({
         help: { type: 'boolean', short: 'h' },
         config: { type: 'string' },
         storage: { type: 'string' },
+        script: { type: 'string' },
         plugin: { type: 'string', multiple: true },
         smtpPort: { type: 'string' }
     }
@@ -28,6 +29,7 @@ const isTrue = value => (value || '').toString().trim().toLowerCase() === 'true'
 
 const configLocation = argv.config || process.env.IMAPKIT_CONFIG;
 const storageLocation = argv.storage || process.env.IMAPKIT_STORAGE;
+const scriptLocation = argv.script || process.env.IMAPKIT_SCRIPT;
 const smtpPort = argv.smtpPort || process.env.IMAPKIT_SMTPPORT;
 const pluginsList = []
     .concat(argv.plugin || process.env.IMAPKIT_PLUGINS || [])
@@ -51,6 +53,10 @@ if (storageLocation) {
     config.storage = JSON.parse(fs.readFileSync(storageLocation, 'utf-8'));
 }
 
+if (scriptLocation) {
+    config.script = JSON.parse(fs.readFileSync(scriptLocation, 'utf-8'));
+}
+
 if (pluginsList.length) {
     config.plugins = pluginsList;
 }

package/dist/cjs/index.d.ts

@@ -1,6 +1,7 @@
 import createServer, { TAG_REGEX, IMAPServer, IMAPConnection } from './server.js';
 export { TAG_REGEX, IMAPServer, IMAPConnection };
 export type { Attribute, ParsedCommand, IMAPResponse, Notification, Callback, CommandHandler, CommandOptions, Plugin, IMAPError, Message, Mailbox, StorageNamespace, UserData, IMAPServerOptions } from './types.js';
+export type { ScriptRule, ScriptContext, ScriptEvent, ScriptBytes, ScriptHandle } from './script.js';
 declare const imapkit: typeof createServer & {
     TAG_REGEX: RegExp;
     IMAPServer: typeof IMAPServer;

package/dist/cjs/plugins/auth-plain.js

@@ -57,7 +57,7 @@ function authPlainPlugin(server) {
                 authenticate(connection, parsed, data, str);
             };
             // Send an empty continuation request to the client
-            connection.write('+ \r\n');
+            connection.sendContinuation('', 'AUTHENTICATE PLAIN');
         }
         else if (parsed.attributes.length === 1 &&
             // second argument must be Base64 string as ATOM

package/dist/cjs/plugins/idle.js

@@ -66,7 +66,7 @@ function idlePlugin(server) {
                 }, 'INVALID IDLE', parsed, data);
             }
         };
-        connection.write('+ idling\r\n');
+        connection.sendContinuation('idling', 'IDLE');
         connection.processNotifications();
         return callback();
     }, { states: command_states_js_1.states.AUTHENTICATED, noArguments: true });

package/dist/cjs/plugins/oauthbearer.js

@@ -169,7 +169,7 @@ function oauthbearerPlugin(server) {
         if (!args.length) {
             // without an initial response the client sends its response after an empty challenge
             readResponse(connection, parsed, data, decoded => authenticate(connection, parsed, data, decoded));
-            connection.write('+ \r\n');
+            connection.sendContinuation('', 'AUTHENTICATE OAUTHBEARER');
             return callback();
         }
         if (args.length !== 1 || !args[0] || args[0].type !== 'ATOM') {

package/dist/cjs/script.d.ts

@@ -0,0 +1,183 @@
+import type { IMAPConnection, IMAPServer } from './server.js';
+import type { IMAPResponse, Transport } from './types.js';
+/**
+ * Scripted faults: rules that make the server deviate from the protocol on purpose, to test how a client
+ * copes with a broken or unusual server. Every rule watches one kind of event, the first rule that matches
+ * an event handles it. Output rules see the exact bytes that are about to go out, after every plugin and the
+ * core changed the response, so they can send anything at all, also what the strict server never would.
+ */
+/** The events a rule can watch */
+export type ScriptEvent = 'greeting' | 'command' | 'input' | 'response' | 'continuation';
+/** Bytes a rule sends, a function gets the context of the event */
+export type ScriptBytes = string | Buffer | ((context: ScriptContext) => string | Buffer);
+/** The event a rule is checked against */
+export interface ScriptContext {
+    event: ScriptEvent;
+    connection: IMAPConnection;
+    /** the number of the connection, 1 for the first one the server accepted */
+    session: number;
+    state: string;
+    /** the authenticated user, or null */
+    user: string | null;
+    /** path of the selected mailbox, or null */
+    mailbox: string | null;
+    /** tag of the command the event belongs to, or null (greeting, unsolicited responses) */
+    tag: string | null;
+    /** name of the command the event belongs to in upper case (`"UID FETCH"`), or null */
+    command: string | null;
+    /** the received line (command, input) or the bytes about to be sent (output events), as a binary string */
+    data: string;
+    /** the description of the response, see `connection.send()` (response and continuation events) */
+    description?: string | null | undefined;
+    /** the response object (response and continuation events sent with `connection.send()`) */
+    response?: IMAPResponse | undefined;
+}
+/** A rule as it is given to `server.script.add()` or the `script` option */
+export interface ScriptRule {
+    on: ScriptEvent;
+    /** command name or names, case-insensitive (`"FETCH"`, `"UID FETCH"`) */
+    command?: string | string[] | undefined;
+    /** command tag, a string matches exactly */
+    tag?: string | RegExp | undefined;
+    /** response description or descriptions (response and continuation events) */
+    description?: string | string[] | undefined;
+    /** connection number or numbers, 1 for the first connection */
+    session?: number | number[] | undefined;
+    user?: string | undefined;
+    state?: string | string[] | undefined;
+    /** path of the selected mailbox */
+    mailbox?: string | undefined;
+    /** only untagged (true) or only tagged (false) responses (response events) */
+    untagged?: boolean | undefined;
+    /** tested against `context.data`, a string is a regular expression source */
+    match?: string | RegExp | undefined;
+    when?: ((context: ScriptContext) => boolean) | undefined;
+    /** the rule fires from the nth matching event on (default 1) */
+    nth?: number | undefined;
+    /** the rule fires this many times at most (default unlimited) */
+    times?: number | undefined;
+    /** changes or replaces the response object before it is compiled (response events) */
+    mutate?: ((response: IMAPResponse, context: ScriptContext) => IMAPResponse | void) | undefined;
+    /** bytes to send instead of the output, or instead of answering the command or input line. `$TAG` in a string is the tag */
+    send?: ScriptBytes | undefined;
+    /** bytes to send before the output (output events) */
+    before?: ScriptBytes | undefined;
+    /** bytes to send after the output (output events) */
+    after?: ScriptBytes | undefined;
+    /** process the command or input line as usual after `send` (command and input events) */
+    run?: boolean | undefined;
+    /** send nothing (output events), or ignore the command or input line */
+    drop?: boolean | undefined;
+    /** milliseconds to wait before the output goes out, or before the command is handled. Later output waits too */
+    delay?: number | undefined;
+    /** write the bytes in pieces of this many octets */
+    chunk?: number | undefined;
+    /** milliseconds between the pieces of `chunk` (default 10) */
+    chunkDelay?: number | undefined;
+    /** send only this many octets of the bytes, then close the connection */
+    truncate?: number | undefined;
+    /** close the connection after the bytes are sent, `"reset"` destroys the socket instead of ending it */
+    close?: boolean | 'reset' | undefined;
+}
+/** A rule that was added, `server.script.add()` returns it */
+export interface ScriptHandle {
+    readonly rule: ScriptRule;
+    /** events that matched the rule, also before `nth` */
+    readonly matched: number;
+    /** events the rule handled */
+    readonly hits: number;
+    remove(): void;
+}
+/** Output that waits in the output queue of a connection, see `IMAPConnection#queueOutput` */
+export interface OutputOperation {
+    data?: Buffer | undefined;
+    delay?: number | undefined;
+    chunk?: number | undefined;
+    chunkDelay?: number | undefined;
+    close?: boolean | 'reset' | undefined;
+    /** the transport layer of the connection when the output was queued */
+    transport?: Transport | null | undefined;
+}
+declare const SET_MATCHERS: readonly ["command", "description", "session", "state", "user", "mailbox"];
+type SetMatcher = (typeof SET_MATCHERS)[number];
+interface Entry extends ScriptHandle {
+    matched: number;
+    hits: number;
+    match: RegExp | null;
+    sets: Partial<Record<SetMatcher, Set<string | number>>>;
+}
+/**
+ * The script of a server, `server.script`
+ */
+export declare class ServerScript {
+    server: IMAPServer;
+    entries: Entry[];
+    /** the events that rules watch, so that an event nobody watches costs one lookup */
+    watched: Set<ScriptEvent>;
+    constructor(server: IMAPServer, rules?: ScriptRule | ScriptRule[] | undefined);
+    /** the rules that were added, in the order they are checked */
+    get rules(): ScriptHandle[];
+    /**
+     * Adds a rule, or a list of rules, after the rules added earlier
+     *
+     * @param {Object|Array} rules Rule or rules
+     * @return {Object|Array} handle or handles with `hits`, `matched` and `remove()`
+     */
+    add(rule: ScriptRule): ScriptHandle;
+    add(rules: ScriptRule[]): ScriptHandle[];
+    add(rules: ScriptRule | ScriptRule[]): ScriptHandle | ScriptHandle[];
+    /** removes every rule */
+    clear(): void;
+    /**
+     * Checks if any rule watches an event, so that the event can skip building a context
+     *
+     * @param {String} event Event name
+     * @return {Boolean} true if a rule watches the event
+     */
+    watches(event: ScriptEvent): boolean;
+    /**
+     * Finds the rule that handles an event of a connection
+     *
+     * @param {Object} connection IMAP connection
+     * @param {String} event Event name
+     * @param {Object} fields `data`, and `tag`, `command`, `description`, `response` where they are known
+     * @return {Object|null} `{ rule, context }`, or null if no rule handles the event
+     */
+    check(connection: IMAPConnection, event: ScriptEvent, fields: Partial<ScriptContext> & {
+        data: string;
+    }): {
+        rule: ScriptRule;
+        context: ScriptContext;
+    } | null;
+    private setEntries;
+    private addEntry;
+    /**
+     * Finds the rule that handles an event and counts it
+     *
+     * @param {Object} context Event context
+     * @return {Object|null} the rule, or null if no rule handles the event
+     */
+    find(context: ScriptContext): ScriptRule | null;
+    private matches;
+}
+/**
+ * Sends output through the rule that handles it: the output itself (unless dropped or replaced with `send`)
+ * between `before` and `after`, cut by `truncate`, with the timing of `delay` and `chunk`
+ *
+ * @param {Object} connection IMAP connection
+ * @param {Object} rule Rule that handles the output
+ * @param {Object} context Event context
+ * @param {Buffer} output The output
+ */
+export declare function sendOutput(connection: IMAPConnection, rule: ScriptRule, context: ScriptContext, output: Buffer): void;
+/**
+ * Handles a command or input line with the rule that matched it: sends the bytes of `send`, closes the connection,
+ * and processes the line as usual with `run` (or when the rule only delays it)
+ *
+ * @param {Object} connection IMAP connection
+ * @param {Object} rule Rule that matched
+ * @param {Object} context Event context
+ * @param {Function} run Processes the line as usual
+ */
+export declare function handleLine(connection: IMAPConnection, rule: ScriptRule, context: ScriptContext, run: () => void): void;
+export {};