mailwoman 0.2.0

package/tokenization/permutate.test.ts

@@ -0,0 +1,368 @@
+/**
+ * @copyright OpenISP, Inc.
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+
+import test from "tape"
+import { Span } from "./Span.js"
+import permutate from "./permutate.js"
+
+function getSpans(spans: Span[], ...indexes: number[]) {
+	return indexes.map((i) => spans[i]!)
+}
+
+test("permutate: simple", (t) => {
+	const spans = Span.connectSiblings(new Span("SoHo", 0), new Span("New", 5), new Span("York", 9), new Span("USA", 14))
+
+	// expected permutations
+	const perm1 = new Span("SoHo New York USA", 0)
+	spans.slice(0, 4).forEach((s) => perm1.graph.add("child", s))
+	perm1.graph.add("child:first", spans[0]!)
+	perm1.graph.add("child:last", spans[3]!)
+
+	const perm2 = new Span("SoHo New York", 0)
+	spans.slice(0, 3).forEach((s) => perm2.graph.add("child", s))
+	perm2.graph.add("child:first", spans[0]!)
+	perm2.graph.add("child:last", spans[2]!)
+
+	const perm3 = new Span("SoHo New", 0)
+	spans.slice(0, 2).forEach((s) => perm3.graph.add("child", s))
+	perm3.graph.add("child:first", spans[0]!)
+	perm3.graph.add("child:last", spans[1]!)
+
+	const perm4 = new Span("SoHo", 0)
+	spans.slice(0, 1).forEach((s) => perm4.graph.add("child", s))
+	perm4.graph.add("child:first", spans[0]!)
+	perm4.graph.add("child:last", spans[0]!)
+
+	const perm5 = new Span("New York USA", 5)
+	spans.slice(1, 4).forEach((s) => perm5.graph.add("child", s))
+	perm5.graph.add("child:first", spans[1]!)
+	perm5.graph.add("child:last", spans[3]!)
+
+	const perm6 = new Span("New York", 5)
+	spans.slice(1, 3).forEach((s) => perm6.graph.add("child", s))
+	perm6.graph.add("child:first", spans[1]!)
+	perm6.graph.add("child:last", spans[2]!)
+
+	const perm7 = new Span("New", 5)
+	spans.slice(1, 2).forEach((s) => perm7.graph.add("child", s))
+	perm7.graph.add("child:first", spans[1]!)
+	perm7.graph.add("child:last", spans[1]!)
+
+	const perm8 = new Span("York USA", 9)
+	spans.slice(2, 4).forEach((s) => perm8.graph.add("child", s))
+	perm8.graph.add("child:first", spans[2]!)
+	perm8.graph.add("child:last", spans[3]!)
+
+	const perm9 = new Span("York", 9)
+	spans.slice(2, 3).forEach((s) => perm9.graph.add("child", s))
+	perm9.graph.add("child:first", spans[2]!)
+	perm9.graph.add("child:last", spans[2]!)
+
+	const perm10 = new Span("USA", 14)
+	spans.slice(3, 4).forEach((s) => perm10.graph.add("child", s))
+	perm10.graph.add("child:first", spans[3]!)
+	perm10.graph.add("child:last", spans[3]!)
+
+	const actual = permutate(spans, 1, 6)
+	t.deepEquals(actual, [perm1, perm2, perm3, perm4, perm5, perm6, perm7, perm8, perm9, perm10])
+	t.end()
+})
+
+test("permutate: tokens contain whitespace", (t) => {
+	const spans = Span.connectSiblings(new Span("SoHo", 0), new Span("New York", 5), new Span("USA", 14))
+
+	// expected permutations
+	const perm1 = new Span("SoHo New York USA", 0)
+	spans.slice(0, 3).forEach((s) => perm1.graph.add("child", s))
+	perm1.graph.add("child:first", spans[0]!)
+	perm1.graph.add("child:last", spans[2]!)
+
+	const perm2 = new Span("SoHo New York", 0)
+	spans.slice(0, 2).forEach((s) => perm2.graph.add("child", s))
+	perm2.graph.add("child:first", spans[0]!)
+	perm2.graph.add("child:last", spans[1]!)
+
+	const perm3 = new Span("SoHo", 0)
+	spans.slice(0, 1).forEach((s) => perm3.graph.add("child", s))
+	perm3.graph.add("child:first", spans[0]!)
+	perm3.graph.add("child:last", spans[0]!)
+
+	const perm4 = new Span("New York USA", 5)
+	spans.slice(1, 3).forEach((s) => perm4.graph.add("child", s))
+	perm4.graph.add("child:first", spans[1]!)
+	perm4.graph.add("child:last", spans[2]!)
+
+	const perm5 = new Span("New York", 5)
+	spans.slice(1, 2).forEach((s) => perm5.graph.add("child", s))
+	perm5.graph.add("child:first", spans[1]!)
+	perm5.graph.add("child:last", spans[1]!)
+
+	const perm6 = new Span("USA", 14)
+	spans.slice(2, 3).forEach((s) => perm6.graph.add("child", s))
+	perm6.graph.add("child:first", spans[2]!)
+	perm6.graph.add("child:last", spans[2]!)
+
+	const actual = permutate(spans, 1, 6)
+	t.deepEquals(actual, [perm1, perm2, perm3, perm4, perm5, perm6])
+	t.end()
+})
+
+test("permutate: smaller window", (t) => {
+	const spans = Span.connectSiblings(new Span("SoHo", 0), new Span("New", 5), new Span("York", 9), new Span("USA", 13))
+
+	// expected permutations
+	const perm1 = new Span("SoHo New", 0)
+	spans.slice(0, 2).forEach((s) => perm1.graph.add("child", s))
+	perm1.graph.add("child:first", spans[0]!)
+	perm1.graph.add("child:last", spans[1]!)
+
+	const perm2 = new Span("SoHo", 0)
+	spans.slice(0, 1).forEach((s) => perm2.graph.add("child", s))
+	perm2.graph.add("child:first", spans[0]!)
+	perm2.graph.add("child:last", spans[0]!)
+
+	const perm3 = new Span("New York", 5)
+	spans.slice(1, 3).forEach((s) => perm3.graph.add("child", s))
+	perm3.graph.add("child:first", spans[1]!)
+	perm3.graph.add("child:last", spans[2]!)
+
+	const perm4 = new Span("New", 5)
+	spans.slice(1, 2).forEach((s) => perm4.graph.add("child", s))
+	perm4.graph.add("child:first", spans[1]!)
+	perm4.graph.add("child:last", spans[1]!)
+
+	const perm5 = new Span("York USA", 9)
+	spans.slice(2, 4).forEach((s) => perm5.graph.add("child", s))
+	perm5.graph.add("child:first", spans[2]!)
+	perm5.graph.add("child:last", spans[3]!)
+
+	const perm6 = new Span("York", 9)
+	spans.slice(2, 3).forEach((s) => perm6.graph.add("child", s))
+	perm6.graph.add("child:first", spans[2]!)
+	perm6.graph.add("child:last", spans[2]!)
+
+	const perm7 = new Span("USA", 13)
+	spans.slice(3, 4).forEach((s) => perm7.graph.add("child", s))
+	perm7.graph.add("child:first", spans[3]!)
+	perm7.graph.add("child:last", spans[3]!)
+
+	const actual = permutate(spans, 1, 2)
+	t.deepEquals(actual, [perm1, perm2, perm3, perm4, perm5, perm6, perm7])
+	t.end()
+})
+
+test("permutate: start/end values", (t) => {
+	// "  SoHo     New  York  "
+	const span1 = new Span("SoHo", 2)
+	span1.end = 6
+	const span2 = new Span("New", 11)
+	span2.end = 14
+	const span3 = new Span("York", 15)
+	span3.end = 19
+	const spans = Span.connectSiblings(span1, span2, span3)
+
+	// expected permutations
+	const perm1 = new Span("SoHo New York", 2)
+	perm1.start = span1.start
+	perm1.end = span3.end
+	spans.slice(0, 3).forEach((s) => perm1.graph.add("child", s))
+	perm1.graph.add("child:first", spans[0]!)
+	perm1.graph.add("child:last", spans[2]!)
+
+	const perm2 = new Span("SoHo New", 2)
+	perm2.start = span1.start
+	perm2.end = span2.end
+	spans.slice(0, 2).forEach((s) => perm2.graph.add("child", s))
+	perm2.graph.add("child:first", spans[0]!)
+	perm2.graph.add("child:last", spans[1]!)
+
+	const perm3 = new Span("SoHo", 2)
+	perm3.start = span1.start
+	perm3.end = span1.end
+	spans.slice(0, 1).forEach((s) => perm3.graph.add("child", s))
+	perm3.graph.add("child:first", spans[0]!)
+	perm3.graph.add("child:last", spans[0]!)
+
+	const perm4 = new Span("New York", 11)
+	perm4.start = span2.start
+	perm4.end = span3.end
+	spans.slice(1, 3).forEach((s) => perm4.graph.add("child", s))
+	perm4.graph.add("child:first", spans[1]!)
+	perm4.graph.add("child:last", spans[2]!)
+
+	const perm5 = new Span("New", 11)
+	perm5.start = span2.start
+	perm5.end = span2.end
+	spans.slice(1, 2).forEach((s) => perm5.graph.add("child", s))
+	perm5.graph.add("child:first", spans[1]!)
+	perm5.graph.add("child:last", spans[1]!)
+
+	const perm6 = new Span("York", 15)
+	perm6.start = span3.start
+	perm6.end = span3.end
+
+	spans.slice(2, 3).forEach((s) => perm6.graph.add("child", s))
+	perm6.graph.add("child:first", spans[2]!)
+	perm6.graph.add("child:last", spans[2]!)
+
+	const actual = permutate(spans, 1, 6)
+	t.deepEquals(actual, [perm1, perm2, perm3, perm4, perm5, perm6])
+	t.end()
+})
+
+test("permutate: relationships", (t) => {
+	const span1 = new Span("SoHo", 0)
+	const span2 = new Span("New", 5)
+	const span3 = new Span("York", 14)
+	const spans = Span.connectSiblings(span1, span2, span3)
+
+	const actual = permutate(spans, 1, 6)
+
+	// Soho New York
+	const perm1 = actual[0]!
+
+	t.true(perm1.graph.findAll("child").includes(span1))
+	t.true(span1.graph.findAll("parent").includes(perm1))
+	t.true(perm1.graph.findAll("child").includes(span2))
+	t.true(span2.graph.findAll("parent").includes(perm1))
+	t.true(perm1.graph.findAll("child").includes(span3))
+	t.true(span3.graph.findAll("parent").includes(perm1))
+
+	// Soho New
+	const perm2 = actual[1]!
+
+	t.true(perm2.graph.findAll("child").includes(span1))
+	t.true(span1.graph.findAll("parent").includes(perm2))
+	t.true(perm2.graph.findAll("child").includes(span2))
+	t.true(span2.graph.findAll("parent").includes(perm2))
+
+	// Soho
+	const perm3 = actual[2]!
+	t.true(perm3.graph.findAll("child").includes(span1))
+	t.true(span1.graph.findAll("parent").includes(perm3))
+
+	// New York
+	const perm4 = actual[3]!
+	t.true(perm4.graph.findAll("child").includes(span2))
+	t.true(span2.graph.findAll("parent").includes(perm4))
+	t.true(perm4.graph.findAll("child").includes(span3))
+	t.true(span3.graph.findAll("parent").includes(perm4))
+
+	// New
+	const perm5 = actual[4]!
+	t.true(perm5.graph.findAll("child").includes(span2))
+	t.true(span2.graph.findAll("parent").includes(perm5))
+
+	// York
+	const perm6 = actual[5]!
+	t.true(perm6.graph.findAll("child").includes(span3))
+	t.true(span3.graph.findAll("parent").includes(perm6))
+
+	t.end()
+})
+
+test("permutate: with hyphen", (t) => {
+	const spans = Span.connectSiblings(new Span("SoHo", 0), new Span("New-York", 5), new Span("USA", 14))
+	spans.push(new Span("New", 5))
+	spans.push(new Span("York", 9))
+
+	// SoHo -> New
+	spans[0]!.graph.add("next", spans[3]!)
+	// New -> York
+	spans[3]!.graph.add("next", spans[4]!)
+	// York -> USA
+	spans[4]!.graph.add("next", spans[2]!)
+
+	// expected permutations
+	const perm1 = new Span("SoHo New-York USA", 0)
+	spans.slice(0, 3).forEach((s) => perm1.graph.add("child", s))
+	perm1.graph.add("child:first", spans[0]!)
+	perm1.graph.add("child:last", spans[2]!)
+
+	const perm2 = new Span("SoHo New-York", 0)
+	spans.slice(0, 2).forEach((s) => perm2.graph.add("child", s))
+	perm2.graph.add("child:first", spans[0]!)
+	perm2.graph.add("child:last", spans[1]!)
+
+	const perm3 = new Span("SoHo New York USA", 0)
+	getSpans(spans, 0, 3, 4, 2).forEach((s) => perm3.graph.add("child", s))
+	perm3.graph.add("child:first", spans[0]!)
+	perm3.graph.add("child:last", spans[2]!)
+
+	const perm4 = new Span("SoHo New York", 0)
+	getSpans(spans, 0, 3, 4).forEach((s) => perm4.graph.add("child", s))
+	perm4.graph.add("child:first", spans[0]!)
+	perm4.graph.add("child:last", spans[4]!)
+
+	const perm5 = new Span("SoHo New", 0)
+	getSpans(spans, 0, 3).forEach((s) => perm5.graph.add("child", s))
+	perm5.graph.add("child:first", spans[0]!)
+	perm5.graph.add("child:last", spans[3]!)
+
+	const perm6 = new Span("SoHo", 0)
+	spans.slice(0, 1).forEach((s) => perm6.graph.add("child", s))
+	perm6.graph.add("child:first", spans[0]!)
+	perm6.graph.add("child:last", spans[0]!)
+
+	const perm7 = new Span("New-York USA", 5)
+	spans.slice(1, 3).forEach((s) => perm7.graph.add("child", s))
+	perm7.graph.add("child:first", spans[1]!)
+	perm7.graph.add("child:last", spans[2]!)
+
+	const perm8 = new Span("New-York", 5)
+	spans.slice(1, 2).forEach((s) => perm8.graph.add("child", s))
+	perm8.graph.add("child:first", spans[1]!)
+	perm8.graph.add("child:last", spans[1]!)
+
+	const perm9 = new Span("USA", 14)
+	spans.slice(2, 3).forEach((s) => perm9.graph.add("child", s))
+	perm9.graph.add("child:first", spans[2]!)
+	perm9.graph.add("child:last", spans[2]!)
+
+	const perm10 = new Span("New York USA", 5)
+	getSpans(spans, 3, 4, 2).forEach((s) => perm10.graph.add("child", s))
+	perm10.graph.add("child:first", spans[3]!)
+	perm10.graph.add("child:last", spans[2]!)
+
+	const perm11 = new Span("New York", 5)
+	getSpans(spans, 3, 4).forEach((s) => perm11.graph.add("child", s))
+	perm11.graph.add("child:first", spans[3]!)
+	perm11.graph.add("child:last", spans[4]!)
+
+	const perm12 = new Span("New", 5)
+	spans.slice(3, 4).forEach((s) => perm12.graph.add("child", s))
+	perm12.graph.add("child:first", spans[3]!)
+	perm12.graph.add("child:last", spans[3]!)
+
+	const perm13 = new Span("York USA", 9)
+	getSpans(spans, 4, 2).forEach((s) => perm13.graph.add("child", s))
+	perm13.graph.add("child:first", spans[4]!)
+	perm13.graph.add("child:last", spans[2]!)
+
+	const perm14 = new Span("York", 9)
+	spans.slice(4, 5).forEach((s) => perm14.graph.add("child", s))
+	perm14.graph.add("child:first", spans[4]!)
+	perm14.graph.add("child:last", spans[4]!)
+
+	const actual = permutate(spans, 1, 10)
+	t.deepEquals(actual, [
+		perm1,
+		perm2,
+		perm3,
+		perm4,
+		perm5,
+		perm6,
+		perm7,
+		perm8,
+		perm9,
+		perm10,
+		perm11,
+		perm12,
+		perm13,
+		perm14,
+	])
+	t.end()
+})

package/tokenization/permutate.ts

@@ -0,0 +1,97 @@
+/**
+ * @copyright OpenISP, Inc.
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+
+import { Span } from "./Span.js"
+const JOIN_CHAR = " "
+
+/**
+ * Produce all the possible token groups from adjacent input tokens (without reordering tokens)
+ *
+ * WindowMin: the minimum amount of tokens which can be returned in a single window windowMax: the
+ * maximum amount of tokens which can be returned in a single window
+ *
+ * Note: we should honor word boundary delimiters (such as comma) when creating permutations ported:
+ * https://github.com/pelias/placeholder/blob/master/lib/permutations.js
+ */
+
+function permutateRec(
+	prevSpan: Span,
+	currentSpan: Span,
+	windowCur: number,
+	windowMin: number,
+	windowMax: number,
+	permutations: Span[]
+) {
+	// Stops when the window is reached
+	if (windowCur > windowMax) {
+		return
+	}
+	// Create new span base on the previous and the next one
+	const span = new Span(prevSpan.body + (prevSpan.body.length > 0 ? JOIN_CHAR : "") + currentSpan.body, prevSpan.start)
+	// Add all children from the previous span to the new one, they will have the same ones + the next one
+	// Add to all children from the previous span the new span as parent + the next one
+	prevSpan.graph.findAll("child").forEach((child) => {
+		span.graph.add("child", child)
+		child.graph.add("parent", span)
+	})
+	span.graph.add("child", currentSpan)
+	currentSpan.graph.add("parent", span)
+
+	const isFirst = span.body === currentSpan.body
+	const isLast = !currentSpan.graph.findOne("next")
+
+	// If span is the first one, s is the first child, otherwise we take the first child of the previous span
+	if (isFirst) {
+		span.graph.add("child:first", currentSpan)
+	} else {
+		span.graph.add("child:first", prevSpan.graph.findOne("child:first")!)
+	}
+
+	span.graph.add("child:last", currentSpan)
+
+	if (isFirst) {
+		span.start = currentSpan.start
+		span.end = currentSpan.end
+	} else {
+		if (currentSpan.start < span.start) {
+			span.start = currentSpan.start
+		}
+		if (currentSpan.end > span.end) {
+			span.end = currentSpan.end
+		}
+	}
+
+	// go through the graph recursively, check all next spans
+	if (!isLast) {
+		currentSpan.graph.findAll("next").forEach((next) => {
+			permutateRec(span, next, windowCur + 1, windowMin, windowMax, permutations)
+		})
+	}
+
+	if (windowMin <= windowCur) {
+		permutations.push(span)
+	}
+}
+
+/**
+ * Produce all the possible token groups from adjacent input tokens (without reordering tokens)
+ */
+function permutate(spans: Span[], windowMin: number, windowMax: number): Span[] {
+	const permutations: Span[] = []
+
+	spans.forEach((span) => {
+		permutateRec(new Span(), span, 1, windowMin, windowMax, permutations)
+	})
+	return permutations
+}
+
+export default permutate
+
+/**
+ * Example: ['soho', 'new', 'york', 'usa'] [ ['soho', 'new', 'york', 'usa'], ['soho', 'new',
+ * 'york'], ['soho', 'new'], ['soho'], ['new', 'york', 'usa'], ['new', 'york'], ['new'], ['york',
+ * 'usa'], ['york'], ['usa'], ]
+ */

package/tokenization/split.test.ts

@@ -0,0 +1,84 @@
+/**
+ * @copyright OpenISP, Inc.
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+
+import deepEqual from "deep-eql"
+import test from "tape"
+import { Span } from "./Span.js"
+import split from "./split.js"
+import * as funcs from "./split_funcs.js"
+
+test("boundary: no commas or quotes", (t) => {
+	const span = new Span("SoHo New York USA")
+	const actual = split(span, funcs.fieldsFuncBoundary)
+
+	t.deepEquals(actual, [new Span("SoHo New York USA")])
+	t.end()
+})
+
+test("boundary: commas", (t) => {
+	const span = new Span("SoHo,,, New York, USA")
+	const actual = split(span, funcs.fieldsFuncBoundary)
+
+	const token1 = new Span("SoHo", 0)
+	const token2 = new Span(" New York", 7)
+	const token3 = new Span(" USA", 17)
+
+	// relationships
+	token1.graph.add("next", token2)
+	token2.graph.add("prev", token1)
+	token2.graph.add("next", token3)
+	token3.graph.add("prev", token2)
+
+	t.true(deepEqual(actual, [token1, token2, token3]))
+	t.end()
+})
+
+test("boundary: quotes", (t) => {
+	const span = new Span('SoHo "New York" USA')
+	const actual = split(span, funcs.fieldsFuncBoundary)
+
+	const token1 = new Span("SoHo ", 0)
+	const token2 = new Span("New York", 6)
+	const token3 = new Span(" USA", 15)
+
+	// relationships
+	token1.graph.add("next", token2)
+	token2.graph.add("prev", token1)
+	token2.graph.add("next", token3)
+	token3.graph.add("prev", token2)
+
+	t.true(deepEqual(actual, [token1, token2, token3]))
+	t.end()
+})
+
+test("whitespace: no whitespace", (t) => {
+	const span = new Span("SoHo")
+	const actual = split(span, funcs.fieldsFuncWhiteSpace)
+
+	t.deepEquals(actual, [new Span("SoHo")])
+	t.end()
+})
+
+test("whitespace: contains whitespace", (t) => {
+	const span = new Span("SoHo\t New York \n USA")
+	const actual = split(span, funcs.fieldsFuncWhiteSpace)
+
+	const token1 = new Span("SoHo", 0)
+	const token2 = new Span("New", 6)
+	const token3 = new Span("York", 10)
+	const token4 = new Span("USA", 17)
+
+	// relationships
+	token1.graph.add("next", token2)
+	token2.graph.add("prev", token1)
+	token2.graph.add("next", token3)
+	token3.graph.add("prev", token2)
+	token3.graph.add("next", token4)
+	token4.graph.add("prev", token3)
+
+	t.true(deepEqual(actual, [token1, token2, token3, token4]))
+	t.end()
+})

package/tokenization/split.ts

@@ -0,0 +1,54 @@
+/**
+ * @copyright OpenISP, Inc.
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+
+import { Span } from "./Span.js"
+
+function split(span: Span, f: (char: string) => boolean): Span[] {
+	// A span is used to record a slice of s of the form s[start:end].
+	// The start index is inclusive and the end index is exclusive.
+	const spans = []
+
+	// Find the field start and end indices.
+	let wasField = false
+	let fromIndex = 0
+
+	// Iterate unicode code points in string
+	for (let i = 0; i < span.body.length; i++) {
+		const char = span.body.charAt(i)
+		if (f(char)) {
+			if (wasField) {
+				spans.push(
+					span.graph
+						.findAll("child")
+						.find((s) => s.start === span.start + fromIndex && s.body === span.body.substring(fromIndex, i)) ||
+						new Span(span.body.substring(fromIndex, i), span.start + fromIndex)
+				)
+				wasField = false
+			}
+		} else if (!wasField) {
+			fromIndex = i
+			wasField = true
+		}
+	}
+
+	// Last field might end at EOF.
+	if (wasField) {
+		spans.push(
+			span.graph
+				.findAll("child")
+				.find(
+					(s) => s.start === span.start + fromIndex && s.body === span.body.substring(fromIndex, span.body.length)
+				) || new Span(span.body.substring(fromIndex, span.body.length), span.start + fromIndex)
+		)
+	}
+
+	// Add siblings to graph
+	Span.connectSiblings(...spans)
+
+	return spans
+}
+
+export default split

package/tokenization/split_funcs.ts

@@ -0,0 +1,45 @@
+/**
+ * @copyright OpenISP, Inc.
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+
+const whitespace = /^\s$/
+const quotes = '"«»‘’‚‛“”„‟‹›⹂「」『』〝〞〟﹁﹂﹃﹄＂＇｢｣'
+
+/**
+ * Predicate to test if a character is a field boundary
+ */
+export function fieldsFuncBoundary(char: string): boolean {
+	switch (char) {
+		case "\n":
+			return true
+		case "\t":
+			return true
+		case ",":
+			return true
+		default:
+			// @todo: this should ideally only work for 'matching pairs' of quotes
+			if (quotes.includes(char)) {
+				return true
+			}
+
+			return false
+	}
+}
+
+// test for any unicode whitespace char including newlines and tabs
+// @todo: is this possible in js without using a regex?
+/**
+ * Predicate to test if a character is a field whitespace.
+ */
+export function fieldsFuncWhiteSpace(char: string): boolean {
+	return whitespace.test(char)
+}
+
+/**
+ * Predicate to test if a character is a hyphen or whitespace.
+ */
+export function fieldsFuncHyphenOrWhiteSpace(char: string): boolean {
+	return char === "-" || char === "/" || fieldsFuncWhiteSpace(char)
+}

package/tokenization/split_funcs_test.ts

@@ -0,0 +1,38 @@
+/**
+ * @copyright OpenISP, Inc.
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+
+import test from "tape"
+import * as funcs from "./split_funcs.js"
+
+test("exports: fieldsFuncBoundary", (t) => {
+	t.deepEquals("function", typeof funcs.fieldsFuncBoundary)
+	t.deepEquals(1, funcs.fieldsFuncBoundary.length)
+	t.end()
+})
+test("exports: fieldsFuncWhiteSpace", (t) => {
+	t.deepEquals("function", typeof funcs.fieldsFuncWhiteSpace)
+	t.deepEquals(1, funcs.fieldsFuncWhiteSpace.length)
+	t.end()
+})
+test("fieldsFuncBoundary", (t) => {
+	t.true(funcs.fieldsFuncBoundary(","))
+	t.true(funcs.fieldsFuncBoundary("\n"))
+	t.true(funcs.fieldsFuncBoundary("\t"))
+	t.true(funcs.fieldsFuncBoundary('"'))
+	t.false(funcs.fieldsFuncBoundary("A"))
+	t.false(funcs.fieldsFuncBoundary("1"))
+	t.end()
+})
+
+test("fieldsFuncWhiteSpace", (t) => {
+	t.true(funcs.fieldsFuncWhiteSpace(" "))
+	t.true(funcs.fieldsFuncWhiteSpace("\xa0")) // non-breaking space
+	t.true(funcs.fieldsFuncWhiteSpace("\t"))
+	t.true(funcs.fieldsFuncWhiteSpace("\n"))
+	t.false(funcs.fieldsFuncWhiteSpace("A"))
+	t.false(funcs.fieldsFuncWhiteSpace("1"))
+	t.end()
+})

package/tsconfig.json

@@ -0,0 +1,11 @@
+{
+	"extends": "@sister.software/tsconfig",
+	"compilerOptions": {
+		"baseUrl": ".",
+		"paths": {
+			"mailwoman/sdk/repo": ["./src/sdk/repo"]
+		}
+	},
+
+	"exclude": ["./out/**/*"]
+}