@engine9/schemas 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.eslintignore +1 -0
- package/.prettierrc +7 -0
- package/AGENTS.md +10 -0
- package/LICENSE +21 -0
- package/README.md +33 -0
- package/channels/email/README.md +83 -0
- package/channels/email/index.js +22 -0
- package/channels/email/search.js +86 -0
- package/channels/email/segments.js +81 -0
- package/eslint.config.js +21 -0
- package/event/README.md +79 -0
- package/event/index.js +17 -0
- package/event/schema.js +111 -0
- package/event/search.js +51 -0
- package/message/index.js +8 -0
- package/message/schema.js +130 -0
- package/message/ui.console.json5 +233 -0
- package/package.json +26 -0
- package/person/index.js +100 -0
- package/person/metrics.js +17 -0
- package/person/schema.js +47 -0
- package/person/transforms/inbound/normalize_field_names.js +16 -0
- package/person/transforms/inbound/upsert_tables.js +21 -0
- package/person/transforms/outbound/appendName.js +32 -0
- package/person/transforms/simpleMap.js +34 -0
- package/person/ui.console.json5 +165 -0
- package/person/upsert.test.js +72 -0
- package/person_address/index.js +30 -0
- package/person_address/normalize.js +232 -0
- package/person_address/normalize.test.js +69 -0
- package/person_address/schema.js +49 -0
- package/person_address/transforms/inbound/normalize.js +23 -0
- package/person_address/transforms/inbound/upsert_tables.js +51 -0
- package/person_address/transforms/outbound/appendAddress.js +87 -0
- package/person_address/ui.console.json5 +31 -0
- package/person_custom/index.js +24 -0
- package/person_custom/transforms/appendRemoteId.js +31 -0
- package/person_custom/transforms/inbound/upsert_tables.js +31 -0
- package/person_email/README.md +82 -0
- package/person_email/index.js +35 -0
- package/person_email/schema.js +49 -0
- package/person_email/search.js +61 -0
- package/person_email/segments.js +16 -0
- package/person_email/transforms/inbound/extract_identifiers.js +33 -0
- package/person_email/transforms/inbound/upsert_tables.js +142 -0
- package/person_email/transforms/outbound/appendEmail.js +32 -0
- package/person_email/transforms/outbound/appendEmailHash.js +20 -0
- package/person_email/transforms/outbound/tables.js +14 -0
- package/person_email/ui.console.json5 +58 -0
- package/person_email/upsert.test.js +126 -0
- package/person_hash/README.md +70 -0
- package/person_hash/hash.js +103 -0
- package/person_hash/hash.test.js +165 -0
- package/person_hash/index.js +37 -0
- package/person_hash/schema.js +54 -0
- package/person_hash/search.js +86 -0
- package/person_hash/transforms/inbound/extract_identifiers.js +70 -0
- package/person_hash/transforms/inbound/upsert_tables.js +105 -0
- package/person_hash/transforms/outbound/appendEmailHash.js +42 -0
- package/person_hash/transforms/outbound/appendPhoneHash.js +42 -0
- package/person_hash/ui.console.json5 +80 -0
- package/person_phone/README.md +79 -0
- package/person_phone/index.js +33 -0
- package/person_phone/schema.js +58 -0
- package/person_phone/search.js +79 -0
- package/person_phone/segments.js +26 -0
- package/person_phone/transforms/inbound/extract_identifiers.js +59 -0
- package/person_phone/transforms/inbound/upsert_tables.js +119 -0
- package/person_phone/transforms/outbound/appendPhoneHash.js +37 -0
- package/person_phone/ui.console.json5 +31 -0
- package/person_phone/upsert.test.js +50 -0
- package/person_remote/index.js +58 -0
- package/person_remote/metrics.js +17 -0
- package/person_remote/schema.js +21 -0
- package/person_remote/transforms/inbound/extract_identifiers.js +34 -0
- package/person_remote/transforms/inbound/upsert_tables.js +92 -0
- package/person_remote/transforms/outbound/appendRemotePersonId.js +59 -0
- package/plugin/index.js +13 -0
- package/plugin/schema.js +70 -0
- package/plugin/settings.js +19 -0
- package/plugin/ui.console.json5 +79 -0
- package/report/index.js +7 -0
- package/report/ui.console.json5 +21 -0
- package/segment/index.js +21 -0
- package/segment/schema.js +81 -0
- package/segment/search.js +64 -0
- package/segment/transforms/inbound/upsert.js +44 -0
- package/segment/ui.console.json5 +142 -0
- package/segment_stats/index.js +16 -0
- package/segment_stats/schema.js +30 -0
- package/source_code/index.js +16 -0
- package/source_code/metrics.js +28 -0
- package/source_code/schema.js +29 -0
- package/source_code/ui.console.json5 +31 -0
- package/stacks/limited-pii/index.js +8 -0
- package/stacks/limited-pii/stack.json +22 -0
- package/stacks/stacks.test.js +27 -0
- package/stacks/standard/index.js +7 -0
- package/stacks/standard/stack.json +20 -0
- package/tag/index.js +13 -0
- package/tag/schema.js +23 -0
- package/task/index.js +13 -0
- package/task/schema.js +54 -0
- package/task/ui.console.json5 +60 -0
- package/timeline/index.js +18 -0
- package/timeline/schema.js +18 -0
- package/timeline/transforms/inbound/upsert_tables.js +22 -0
- package/timeline/ui.console.json5 +31 -0
- package/transaction/contact_details/index.js +11 -0
- package/transaction/contact_details/schema.js +22 -0
- package/transaction/core/README.md +79 -0
- package/transaction/core/attribute.test.js +231 -0
- package/transaction/core/index.js +31 -0
- package/transaction/core/metrics.js +20 -0
- package/transaction/core/schema.js +45 -0
- package/transaction/core/search.js +87 -0
- package/transaction/core/segments.js +14 -0
- package/transaction/core/transforms/appendTransactionSummary.js +101 -0
- package/transaction/core/transforms/attribute.js +327 -0
- package/transaction/core/transforms/inbound/upsert_tables.js +57 -0
- package/utilities/limited-pii/index.js +14 -0
- package/utilities/limited-pii/settings.js +12 -0
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export default (async function (batch, { PERSON_EMAIL_TABLE }) {
|
|
2
|
+
const emails = await PERSON_EMAIL_TABLE.select(['person_id', 'email', 'type']).whereIn(
|
|
3
|
+
'person_id',
|
|
4
|
+
batch.map((d) => d.id)
|
|
5
|
+
);
|
|
6
|
+
const map = {};
|
|
7
|
+
emails.forEach((r) => {
|
|
8
|
+
map[r.person_id] = (map[r.person_id] || []).concat({ email: r.email, type: r.type });
|
|
9
|
+
});
|
|
10
|
+
batch.forEach((r) => {
|
|
11
|
+
r.emails = map[r.id] || [];
|
|
12
|
+
});
|
|
13
|
+
return batch;
|
|
14
|
+
});
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
{
|
|
2
|
+
routes: {
|
|
3
|
+
'person/:id/*': {
|
|
4
|
+
components: {
|
|
5
|
+
sidebar: {
|
|
6
|
+
items: [
|
|
7
|
+
{
|
|
8
|
+
component: 'Link',
|
|
9
|
+
properties: {
|
|
10
|
+
label: 'Emails',
|
|
11
|
+
to: 'emails'
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
]
|
|
15
|
+
},
|
|
16
|
+
main: [
|
|
17
|
+
{
|
|
18
|
+
path: 'emails',
|
|
19
|
+
label: 'Emails',
|
|
20
|
+
component: 'RecordTable',
|
|
21
|
+
properties: {
|
|
22
|
+
table: 'person_email',
|
|
23
|
+
conditions: [{ eql: 'person_id={{id}}' }],
|
|
24
|
+
columns: [
|
|
25
|
+
{
|
|
26
|
+
title: 'Id',
|
|
27
|
+
dataIndex: 'id',
|
|
28
|
+
sorter: true,
|
|
29
|
+
width: '20%'
|
|
30
|
+
},
|
|
31
|
+
{
|
|
32
|
+
title: 'Email',
|
|
33
|
+
template: '{{email}}',
|
|
34
|
+
width: '40%'
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
title: 'Type',
|
|
38
|
+
template: '{{email_type}}',
|
|
39
|
+
width: '10%'
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
title: 'Subscription Status',
|
|
43
|
+
template: '{{subscription_status}}',
|
|
44
|
+
width: '15%'
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
title: 'Modified',
|
|
48
|
+
template: '{{date modified_at}}',
|
|
49
|
+
width: '15%'
|
|
50
|
+
}
|
|
51
|
+
]
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
]
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import assert from 'node:assert/strict';
|
|
2
|
+
import { describe, it } from 'node:test';
|
|
3
|
+
import upsertEmails from './transforms/inbound/upsert_tables.js';
|
|
4
|
+
|
|
5
|
+
describe('person_email upsert_tables', () => {
|
|
6
|
+
it('merges duplicate email rows in one batch', async () => {
|
|
7
|
+
const tablesToUpsert = {};
|
|
8
|
+
await upsertEmails.transform({
|
|
9
|
+
batch: [
|
|
10
|
+
{ person_id: 1, email: 'donor@example.com', input_id: 'input-a' },
|
|
11
|
+
{ person_id: 1, email: 'donor@example.com', input_id: 'input-b' }
|
|
12
|
+
],
|
|
13
|
+
databaseEmails: [],
|
|
14
|
+
tablesToUpsert
|
|
15
|
+
});
|
|
16
|
+
assert.equal(tablesToUpsert.person_email.length, 1);
|
|
17
|
+
assert.equal(tablesToUpsert.person_email[0].source_input_id, 'input-a');
|
|
18
|
+
assert.ok('email_hash_v1' in tablesToUpsert.person_email[0], 'email_hash_v1 key must always exist');
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
it('last batch row wins for subscription_status (Unsub then Sub in one file)', async () => {
|
|
22
|
+
const tablesToUpsert = {};
|
|
23
|
+
await upsertEmails.transform({
|
|
24
|
+
batch: [
|
|
25
|
+
{
|
|
26
|
+
person_id: 2,
|
|
27
|
+
email: 'toggle@example.com',
|
|
28
|
+
input_id: 'input-a',
|
|
29
|
+
entry_type: 'EMAIL_UNSUBSCRIBE',
|
|
30
|
+
subscription_status: 'Unsubscribed'
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
person_id: 2,
|
|
34
|
+
email: 'toggle@example.com',
|
|
35
|
+
input_id: 'input-b',
|
|
36
|
+
subscription_status: 'Subscribed'
|
|
37
|
+
}
|
|
38
|
+
],
|
|
39
|
+
databaseEmails: [],
|
|
40
|
+
tablesToUpsert
|
|
41
|
+
});
|
|
42
|
+
assert.equal(tablesToUpsert.person_email.length, 1);
|
|
43
|
+
assert.equal(tablesToUpsert.person_email[0].subscription_status, 'Subscribed');
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
it('last batch row wins for subscription_status (Sub then Unsub in one file)', async () => {
|
|
47
|
+
const tablesToUpsert = {};
|
|
48
|
+
await upsertEmails.transform({
|
|
49
|
+
batch: [
|
|
50
|
+
{
|
|
51
|
+
person_id: 3,
|
|
52
|
+
email: 'toggle2@example.com',
|
|
53
|
+
input_id: 'input-a',
|
|
54
|
+
subscription_status: 'Subscribed'
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
person_id: 3,
|
|
58
|
+
email: 'toggle2@example.com',
|
|
59
|
+
input_id: 'input-b',
|
|
60
|
+
entry_type: 'EMAIL_UNSUBSCRIBE',
|
|
61
|
+
subscription_status: 'Unsubscribed'
|
|
62
|
+
}
|
|
63
|
+
],
|
|
64
|
+
databaseEmails: [],
|
|
65
|
+
tablesToUpsert
|
|
66
|
+
});
|
|
67
|
+
assert.equal(tablesToUpsert.person_email.length, 1);
|
|
68
|
+
assert.equal(tablesToUpsert.person_email[0].subscription_status, 'Unsubscribed');
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
it('queues email_hash_v1 as blank string when missing on some input rows', async () => {
|
|
72
|
+
const tablesToUpsert = {};
|
|
73
|
+
await upsertEmails.transform({
|
|
74
|
+
batch: [
|
|
75
|
+
{ person_id: 10, email: 'a@example.com', input_id: 'input-a', email_hash_v1: 'hash-a' },
|
|
76
|
+
{ person_id: 11, email: 'b@example.com', input_id: 'input-b' }
|
|
77
|
+
],
|
|
78
|
+
databaseEmails: [],
|
|
79
|
+
tablesToUpsert
|
|
80
|
+
});
|
|
81
|
+
assert.equal(tablesToUpsert.person_email.length, 2);
|
|
82
|
+
const rowA = tablesToUpsert.person_email.find((r) => r.person_id === 10);
|
|
83
|
+
const rowB = tablesToUpsert.person_email.find((r) => r.person_id === 11);
|
|
84
|
+
assert.equal(rowA.email_hash_v1, 'hash-a');
|
|
85
|
+
assert.equal(rowB.email_hash_v1, '', 'missing email_hash_v1 should normalize to blank string');
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
it('normalizes type and email_type onto the same email_type key', async () => {
|
|
89
|
+
const tablesToUpsert = {};
|
|
90
|
+
await upsertEmails.transform({
|
|
91
|
+
batch: [
|
|
92
|
+
{ person_id: 1, email: 'a@example.com', input_id: 'input-a', type: 'Work' },
|
|
93
|
+
{ person_id: 2, email: 'b@example.com', input_id: 'input-a', email_type: 'Other' }
|
|
94
|
+
],
|
|
95
|
+
databaseEmails: [],
|
|
96
|
+
tablesToUpsert
|
|
97
|
+
});
|
|
98
|
+
assert.deepEqual(
|
|
99
|
+
tablesToUpsert.person_email.map((row) => row.email_type),
|
|
100
|
+
['Work', 'Other']
|
|
101
|
+
);
|
|
102
|
+
for (const row of tablesToUpsert.person_email) {
|
|
103
|
+
assert.equal('type' in row, false);
|
|
104
|
+
assert.equal('email_type' in row, true);
|
|
105
|
+
}
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
it('keeps the stored email_type when an update does not send one', async () => {
|
|
109
|
+
const tablesToUpsert = {};
|
|
110
|
+
await upsertEmails.transform({
|
|
111
|
+
batch: [{ person_id: 1, email: 'a@example.com', input_id: 'input-a', subscription_status: 'Subscribed' }],
|
|
112
|
+
databaseEmails: [
|
|
113
|
+
{
|
|
114
|
+
id: 9,
|
|
115
|
+
person_id: 1,
|
|
116
|
+
email: 'a@example.com',
|
|
117
|
+
email_type: 'Work',
|
|
118
|
+
subscription_status: 'Subscribed',
|
|
119
|
+
source_input_id: 'input-a'
|
|
120
|
+
}
|
|
121
|
+
],
|
|
122
|
+
tablesToUpsert
|
|
123
|
+
});
|
|
124
|
+
assert.equal(tablesToUpsert.person_email[0].email_type, 'Work');
|
|
125
|
+
});
|
|
126
|
+
});
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# Person Hash Schema Plugin
|
|
2
|
+
|
|
3
|
+
`@engine9/schemas/person_hash` stores **pseudonymous match keys** for email and phone — SHA-256 (`*_hash_v1`) and MD5 (`*_hash_md5`) — without storing the plaintext.
|
|
4
|
+
|
|
5
|
+
It is **not** part of the standard stack. It ships in `@engine9/schemas/stacks/limited-pii`, or install it explicitly (once per account — `unique: true`). Once installed, core weaves its transforms into the inbound people pipeline automatically.
|
|
6
|
+
|
|
7
|
+
Warehouse plugin setting `exclude_pii` on `@engine9/schemas/utilities/limited-pii` makes `installDefaultPlugins()` prefer the limited-pii stack over `default_stack` and refuses `@engine9/schemas/stacks/standard`, `person_email`, `person_phone`, and `person_address` even if those plugins are already installed. That does **not** uninstall leftover plaintext tables. Install the utility plugin on demand; it is not part of the limited-pii stack.
|
|
8
|
+
|
|
9
|
+
It does **not** depend on `person_email` or `person_phone`. Those plugins may be installed alongside it; this schema plugin never writes to their tables.
|
|
10
|
+
|
|
11
|
+
## Install
|
|
12
|
+
|
|
13
|
+
```javascript
|
|
14
|
+
await pluginWorker.install({ path: '@engine9/schemas/person_hash' });
|
|
15
|
+
// or the PII-free stack
|
|
16
|
+
await pluginWorker.installDefaultPlugins({ path: '@engine9/schemas/stacks/limited-pii' });
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
A second install of the same path reuses the existing plugin row.
|
|
20
|
+
|
|
21
|
+
## Inbound pipeline slots
|
|
22
|
+
|
|
23
|
+
`metadata.inbound` declares where the transforms run; core reads it from the installed plugin row and never hardcodes this path:
|
|
24
|
+
|
|
25
|
+
```javascript
|
|
26
|
+
inbound: {
|
|
27
|
+
id: ['extractContactHashes'], // with the other identifier extracts, before person_id assignment
|
|
28
|
+
upsert: ['upsertPersonHash'] // with the other table upserts, after person_id assignment
|
|
29
|
+
}
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
How slots and the weaver work: `@engine9/core/lib/peoplePipeline/README.md`.
|
|
33
|
+
|
|
34
|
+
To see the woven chain for an account: `personWorker.getInboundTransforms({ pluginId, describe: true })` (CLI: `e9 personworker getInboundTransforms --plugin_id=<plugin_id> --describe=true`). On a limited-pii account the listing has `person_hash` on `id` and `upsert` and no `person_email` / `person_phone` / `person_address` lines. Drop a step for one job with `omit_transforms`.
|
|
35
|
+
|
|
36
|
+
## Data Model
|
|
37
|
+
|
|
38
|
+
### `person_hash_email`
|
|
39
|
+
|
|
40
|
+
Primary key: `(person_id, email_hash_v1)`.
|
|
41
|
+
|
|
42
|
+
| Field | Type | Description |
|
|
43
|
+
| --- | --- | --- |
|
|
44
|
+
| `person_id` | `person_id` | Person that owns the hash. |
|
|
45
|
+
| `email_hash_v1` | `hash` | SHA-256 of the trimmed, lowercased email. Used as the person identifier type `email_hash_v1`. |
|
|
46
|
+
| `email_hash_md5` | `hash` | Legacy MD5 of the trimmed, **uppercased** email. Kept for some political matching contexts; empty when the inbound row had a SHA-256 but no plaintext. |
|
|
47
|
+
| `source_input_id` | `foreign_uuid` | Input that originally created the row. Existing rows keep their original source. |
|
|
48
|
+
| `created_at` / `modified_at` | timestamps | |
|
|
49
|
+
|
|
50
|
+
### `person_hash_phone`
|
|
51
|
+
|
|
52
|
+
Primary key: `(person_id, phone_hash_v1)`. Same shape with `phone_hash_v1` / `phone_hash_md5`. Phones are normalized like `person_phone` (digits + optional `+1` for US 10-digit numbers) before hashing.
|
|
53
|
+
|
|
54
|
+
## Inbound Behavior
|
|
55
|
+
|
|
56
|
+
The `extractContactHashes` transform hashes `email` / `phone` (or mobile/cell aliases) when present, or accepts an existing `email_hash_v1` / `phone_hash_v1`. It pushes those SHA-256 values as person identifiers. It does not write email or phone onto the hash tables.
|
|
57
|
+
|
|
58
|
+
The `upsertPersonHash` transform writes only hash columns to `person_hash_email` / `person_hash_phone`.
|
|
59
|
+
|
|
60
|
+
## Outbound Behavior
|
|
61
|
+
|
|
62
|
+
- `appendEmailHash` / `appendPhoneHash` attach hashes from plaintext on the row, or from the hash tables by `person_id`. They do not append email or phone.
|
|
63
|
+
|
|
64
|
+
## Search
|
|
65
|
+
|
|
66
|
+
`PersonWorker.search` `emails` / `phones` match `person_email` / `person_phone` when those tables exist, and also hash the query (or accept SHA-256 / MD5 hex) against `person_hash_email` / `person_hash_phone`. If the caller passed emails or phones and neither table exists, the clause matches nobody (`1=0`). Related summaries include `email_hashes` / `phone_hashes` when those tables are present.
|
|
67
|
+
|
|
68
|
+
The plugin also exports `search.emailHashes` / `search.phoneHashes` for the plugin search tree (`@engine9/schemas/person_hash:search:emailHashes`).
|
|
69
|
+
|
|
70
|
+
Default warehouse export includes the hash tables (export skips them when they do not exist). When utilities/limited-pii `exclude_pii` is set and `tables` is not explicit, `person_email` / `person_phone` / `person_address` are omitted from that default list.
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import nodecrypto from 'node:crypto';
|
|
2
|
+
|
|
3
|
+
const { createHash } = nodecrypto;
|
|
4
|
+
|
|
5
|
+
export const MIN_EMAIL_LENGTH = 5;
|
|
6
|
+
export const MIN_PHONE_LENGTH = 8;
|
|
7
|
+
|
|
8
|
+
/** SHA-256 hex digest of an empty string — must never be stored as an identifier. */
|
|
9
|
+
export const BLANK_SHA256_HEX = createHash('sha256').update('').digest('hex');
|
|
10
|
+
/** MD5 hex digest of an empty string — must never be stored as a hash. */
|
|
11
|
+
export const BLANK_MD5_HEX = createHash('md5').update('').digest('hex');
|
|
12
|
+
|
|
13
|
+
export function digestHex(algorithm, value) {
|
|
14
|
+
return createHash(algorithm).update(String(value), 'utf8').digest('hex');
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function isBlankSha256(value) {
|
|
18
|
+
return Boolean(value) && value === BLANK_SHA256_HEX;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export function isBlankMd5(value) {
|
|
22
|
+
return Boolean(value) && value === BLANK_MD5_HEX;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Trim + lowercase. Matching and `email_hash_v1` always use this form;
|
|
27
|
+
* callers that persist plaintext should keep the original casing.
|
|
28
|
+
* (`email_hash_md5` uses the uppercased form — see `hashEmail`.)
|
|
29
|
+
*/
|
|
30
|
+
export function normalizeEmail(email) {
|
|
31
|
+
if (email == null) return '';
|
|
32
|
+
return String(email).trim().toLowerCase();
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* SHA-256 (`email_hash_v1`) of the trimmed, lowercased email (industry-standard match key).
|
|
37
|
+
* MD5 (`email_hash_md5`) of the trimmed, *uppercased* email — legacy; kept for some
|
|
38
|
+
* political matching contexts that historically hashed uppercase addresses.
|
|
39
|
+
* Returns null when the email is missing or shorter than MIN_EMAIL_LENGTH.
|
|
40
|
+
*/
|
|
41
|
+
export function hashEmail(email) {
|
|
42
|
+
const hashable = normalizeEmail(email);
|
|
43
|
+
if (hashable.length < MIN_EMAIL_LENGTH) return null;
|
|
44
|
+
return {
|
|
45
|
+
email_hash_v1: digestHex('sha256', hashable),
|
|
46
|
+
email_hash_md5: digestHex('md5', hashable.toUpperCase())
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Prefer `phone`, then mobile/cell aliases used on inbound person rows.
|
|
52
|
+
*/
|
|
53
|
+
export function resolvePhoneFromRow(row) {
|
|
54
|
+
if (!row) return '';
|
|
55
|
+
if (row.phone) return row.phone;
|
|
56
|
+
return row.cell || row.mobile || row.mobile_phone || '';
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Strip to digits and `+`. US 10-digit numbers get `+1`; 11-digit numbers
|
|
61
|
+
* starting with `1` get a leading `+`. Numbers already prefixed with `+`
|
|
62
|
+
* are left as-is. Returns '' when the cleaned value is too short.
|
|
63
|
+
*/
|
|
64
|
+
export function normalizePhone(phone) {
|
|
65
|
+
let cleaned = String(phone || '')
|
|
66
|
+
.replace(/[^0-9+]*/g, '')
|
|
67
|
+
.trim();
|
|
68
|
+
if (!cleaned || cleaned.length < MIN_PHONE_LENGTH) return '';
|
|
69
|
+
if (cleaned.indexOf('+') < 0) {
|
|
70
|
+
if (cleaned.length === 10) cleaned = `+1${cleaned}`;
|
|
71
|
+
else if (cleaned.length === 11 && cleaned.slice(0, 1) === '1') cleaned = `+${cleaned}`;
|
|
72
|
+
}
|
|
73
|
+
return cleaned;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* SHA-256 (`phone_hash_v1`) and MD5 (`phone_hash_md5`) of the normalized phone.
|
|
78
|
+
* Returns null when the phone is missing or too short.
|
|
79
|
+
*/
|
|
80
|
+
export function hashPhone(phone) {
|
|
81
|
+
const normalized = normalizePhone(phone);
|
|
82
|
+
if (!normalized) return null;
|
|
83
|
+
return {
|
|
84
|
+
phone: normalized,
|
|
85
|
+
phone_hash_v1: digestHex('sha256', normalized),
|
|
86
|
+
phone_hash_md5: digestHex('md5', normalized)
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export default {
|
|
91
|
+
MIN_EMAIL_LENGTH,
|
|
92
|
+
MIN_PHONE_LENGTH,
|
|
93
|
+
BLANK_SHA256_HEX,
|
|
94
|
+
BLANK_MD5_HEX,
|
|
95
|
+
digestHex,
|
|
96
|
+
isBlankSha256,
|
|
97
|
+
isBlankMd5,
|
|
98
|
+
normalizeEmail,
|
|
99
|
+
hashEmail,
|
|
100
|
+
resolvePhoneFromRow,
|
|
101
|
+
normalizePhone,
|
|
102
|
+
hashPhone
|
|
103
|
+
};
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
import assert from 'node:assert/strict';
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
3
|
+
import { describe, it } from 'node:test';
|
|
4
|
+
import {
|
|
5
|
+
BLANK_MD5_HEX,
|
|
6
|
+
BLANK_SHA256_HEX,
|
|
7
|
+
hashEmail,
|
|
8
|
+
hashPhone,
|
|
9
|
+
normalizeEmail,
|
|
10
|
+
normalizePhone,
|
|
11
|
+
resolvePhoneFromRow
|
|
12
|
+
} from './hash.js';
|
|
13
|
+
import extractIdentifiers from './transforms/inbound/extract_identifiers.js';
|
|
14
|
+
import upsertHashes from './transforms/inbound/upsert_tables.js';
|
|
15
|
+
|
|
16
|
+
function sha256(value) {
|
|
17
|
+
return createHash('sha256').update(value, 'utf8').digest('hex');
|
|
18
|
+
}
|
|
19
|
+
function md5(value) {
|
|
20
|
+
return createHash('md5').update(value, 'utf8').digest('hex');
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
describe('person_hash helpers', () => {
|
|
24
|
+
it('hashes trimmed lowercased email with sha256 and uppercased with md5', () => {
|
|
25
|
+
const hashes = hashEmail(' TeSt@Example.COM ');
|
|
26
|
+
assert.equal(hashes.email_hash_v1, sha256('test@example.com'));
|
|
27
|
+
assert.equal(hashes.email_hash_md5, md5('TEST@EXAMPLE.COM'));
|
|
28
|
+
assert.equal(normalizeEmail(' TeSt@Example.COM '), 'test@example.com');
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
it('rejects short emails', () => {
|
|
32
|
+
assert.equal(hashEmail('a@b'), null);
|
|
33
|
+
assert.equal(hashEmail(''), null);
|
|
34
|
+
assert.equal(hashEmail(null), null);
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
it('normalizes US phones and hashes the E.164 form', () => {
|
|
38
|
+
const hashes = hashPhone('202-555-0143');
|
|
39
|
+
assert.equal(hashes.phone, '+12025550143');
|
|
40
|
+
assert.equal(hashes.phone_hash_v1, sha256('+12025550143'));
|
|
41
|
+
assert.equal(hashes.phone_hash_md5, md5('+12025550143'));
|
|
42
|
+
assert.equal(normalizePhone('+12025550143'), '+12025550143');
|
|
43
|
+
assert.equal(normalizePhone('12025550143'), '+12025550143');
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
it('rejects short phones', () => {
|
|
47
|
+
assert.equal(hashPhone('555-1212'), null);
|
|
48
|
+
assert.equal(normalizePhone('555-1212'), '');
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
it('resolves mobile aliases', () => {
|
|
52
|
+
assert.equal(resolvePhoneFromRow({ mobile_phone: '2025550143' }), '2025550143');
|
|
53
|
+
assert.equal(resolvePhoneFromRow({ phone: '1', cell: '2' }), '1');
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
it('exposes blank digests', () => {
|
|
57
|
+
assert.equal(BLANK_SHA256_HEX, sha256(''));
|
|
58
|
+
assert.equal(BLANK_MD5_HEX, md5(''));
|
|
59
|
+
});
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
describe('person_hash extract_identifiers', () => {
|
|
63
|
+
it('extracts identifiers from plaintext and does not require person_email', async () => {
|
|
64
|
+
const batch = [{ email: 'Alice@Example.com', phone: '202-555-0143' }];
|
|
65
|
+
await extractIdentifiers.transform({ batch });
|
|
66
|
+
assert.equal(batch[0].email_hash_v1, sha256('alice@example.com'));
|
|
67
|
+
assert.equal(batch[0].email_hash_md5, md5('ALICE@EXAMPLE.COM'));
|
|
68
|
+
assert.equal(batch[0].phone, '+12025550143');
|
|
69
|
+
assert.equal(batch[0].phone_hash_v1, sha256('+12025550143'));
|
|
70
|
+
assert.deepEqual(
|
|
71
|
+
batch[0].identifiers.map((id) => id.type),
|
|
72
|
+
['email_hash_v1', 'phone_hash_v1']
|
|
73
|
+
);
|
|
74
|
+
assert.equal(batch[0].identifiers[0].path, 'person_hash_email');
|
|
75
|
+
assert.equal(batch[0].identifiers[1].path, 'person_hash_phone');
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
it('accepts hash-only inbound without computing md5', async () => {
|
|
79
|
+
const emailHash = sha256('only-hash@example.com');
|
|
80
|
+
const batch = [{ email_hash_v1: emailHash }];
|
|
81
|
+
await extractIdentifiers.transform({ batch });
|
|
82
|
+
assert.equal(batch[0].email_hash_v1, emailHash);
|
|
83
|
+
assert.equal(batch[0].email_hash_md5, undefined);
|
|
84
|
+
assert.equal(batch[0].identifiers[0].value, emailHash);
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
it('keeps an existing phone hash when the phone value is too short', async () => {
|
|
88
|
+
const phoneHash = sha256('+12025550143');
|
|
89
|
+
const batch = [{ phone: '123', phone_hash_v1: phoneHash }];
|
|
90
|
+
await extractIdentifiers.transform({ batch });
|
|
91
|
+
assert.equal(batch[0].phone_hash_v1, phoneHash);
|
|
92
|
+
assert.equal(batch[0].identifiers[0].type, 'phone_hash_v1');
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
it('does not push blank hashes', async () => {
|
|
96
|
+
const batch = [{ email_hash_v1: BLANK_SHA256_HEX, phone_hash_v1: BLANK_SHA256_HEX }];
|
|
97
|
+
await extractIdentifiers.transform({ batch });
|
|
98
|
+
assert.equal((batch[0].identifiers || []).length, 0);
|
|
99
|
+
});
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
describe('person_hash upsert', () => {
|
|
103
|
+
it('writes hash columns only and preserves source_input_id', async () => {
|
|
104
|
+
const emailHash = sha256('a@example.com');
|
|
105
|
+
const tablesToUpsert = {};
|
|
106
|
+
await upsertHashes.transform({
|
|
107
|
+
batch: [
|
|
108
|
+
{
|
|
109
|
+
person_id: 9,
|
|
110
|
+
input_id: 'new-input',
|
|
111
|
+
email: 'a@example.com',
|
|
112
|
+
email_hash_v1: emailHash,
|
|
113
|
+
email_hash_md5: md5('A@EXAMPLE.COM'),
|
|
114
|
+
given_name: 'Ada'
|
|
115
|
+
}
|
|
116
|
+
],
|
|
117
|
+
databaseEmailHashes: [
|
|
118
|
+
{
|
|
119
|
+
person_id: 9,
|
|
120
|
+
email_hash_v1: emailHash,
|
|
121
|
+
email_hash_md5: '',
|
|
122
|
+
source_input_id: 'original-input'
|
|
123
|
+
}
|
|
124
|
+
],
|
|
125
|
+
databasePhoneHashes: [],
|
|
126
|
+
tablesToUpsert
|
|
127
|
+
});
|
|
128
|
+
assert.deepEqual(Object.keys(tablesToUpsert), ['person_hash_email']);
|
|
129
|
+
assert.deepEqual(tablesToUpsert.person_hash_email[0], {
|
|
130
|
+
person_id: 9,
|
|
131
|
+
email_hash_v1: emailHash,
|
|
132
|
+
email_hash_md5: md5('A@EXAMPLE.COM'),
|
|
133
|
+
source_input_id: 'original-input'
|
|
134
|
+
});
|
|
135
|
+
assert.equal('email' in tablesToUpsert.person_hash_email[0], false);
|
|
136
|
+
assert.equal('given_name' in tablesToUpsert.person_hash_email[0], false);
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
it('skips rows without person_id or usable hashes', async () => {
|
|
140
|
+
const tablesToUpsert = {};
|
|
141
|
+
await upsertHashes.transform({
|
|
142
|
+
batch: [{ email_hash_v1: sha256('x@y.com') }, { person_id: 1 }],
|
|
143
|
+
databaseEmailHashes: [],
|
|
144
|
+
databasePhoneHashes: [],
|
|
145
|
+
tablesToUpsert
|
|
146
|
+
});
|
|
147
|
+
assert.deepEqual(tablesToUpsert, {});
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
it('merges duplicate hash rows in one batch', async () => {
|
|
151
|
+
const emailHash = sha256('dup@example.com');
|
|
152
|
+
const tablesToUpsert = {};
|
|
153
|
+
await upsertHashes.transform({
|
|
154
|
+
batch: [
|
|
155
|
+
{ person_id: 5, input_id: 'input-a', email_hash_v1: emailHash, email_hash_md5: md5('DUP@EXAMPLE.COM') },
|
|
156
|
+
{ person_id: 5, input_id: 'input-b', email_hash_v1: emailHash, email_hash_md5: md5('DUP@EXAMPLE.COM') }
|
|
157
|
+
],
|
|
158
|
+
databaseEmailHashes: [],
|
|
159
|
+
databasePhoneHashes: [],
|
|
160
|
+
tablesToUpsert
|
|
161
|
+
});
|
|
162
|
+
assert.equal(tablesToUpsert.person_hash_email.length, 1);
|
|
163
|
+
assert.equal(tablesToUpsert.person_hash_email[0].source_input_id, 'input-a');
|
|
164
|
+
});
|
|
165
|
+
});
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import schema from './schema.js';
|
|
2
|
+
import search from './search.js';
|
|
3
|
+
import extractContactHashes from './transforms/inbound/extract_identifiers.js';
|
|
4
|
+
import upsertPersonHash from './transforms/inbound/upsert_tables.js';
|
|
5
|
+
import appendEmailHash from './transforms/outbound/appendEmailHash.js';
|
|
6
|
+
import appendPhoneHash from './transforms/outbound/appendPhoneHash.js';
|
|
7
|
+
|
|
8
|
+
const metadata = {
|
|
9
|
+
name: '@engine9/schemas/person_hash',
|
|
10
|
+
unique: true,
|
|
11
|
+
dependencies: {
|
|
12
|
+
'@engine9/schemas/person': '>=1.9.0'
|
|
13
|
+
},
|
|
14
|
+
// Inbound people pipeline slots -> transform export keys. Core weaves these in
|
|
15
|
+
// whenever this plugin is installed; it never hardcodes this path.
|
|
16
|
+
inbound: {
|
|
17
|
+
id: ['extractContactHashes'],
|
|
18
|
+
upsert: ['upsertPersonHash']
|
|
19
|
+
}
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
export const transforms = {
|
|
23
|
+
extractContactHashes,
|
|
24
|
+
upsertPersonHash,
|
|
25
|
+
appendEmailHash,
|
|
26
|
+
appendPhoneHash
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
export { metadata };
|
|
30
|
+
export { schema };
|
|
31
|
+
export { search };
|
|
32
|
+
export default {
|
|
33
|
+
metadata,
|
|
34
|
+
schema,
|
|
35
|
+
transforms,
|
|
36
|
+
search
|
|
37
|
+
};
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
export const tables = [
|
|
2
|
+
{
|
|
3
|
+
name: 'person_hash_email',
|
|
4
|
+
columns: {
|
|
5
|
+
person_id: 'person_id',
|
|
6
|
+
email_hash_v1: {
|
|
7
|
+
type: 'hash',
|
|
8
|
+
description: 'SHA-256 hex of the trimmed, lowercased email (identity match key)'
|
|
9
|
+
},
|
|
10
|
+
email_hash_md5: {
|
|
11
|
+
type: 'hash',
|
|
12
|
+
nullable: true,
|
|
13
|
+
default_value: '',
|
|
14
|
+
description:
|
|
15
|
+
'Legacy MD5 hex of the trimmed, uppercased email (political match contexts); empty when plaintext was not available'
|
|
16
|
+
},
|
|
17
|
+
source_input_id: 'foreign_uuid',
|
|
18
|
+
created_at: 'created_at',
|
|
19
|
+
modified_at: 'modified_at'
|
|
20
|
+
},
|
|
21
|
+
indexes: [
|
|
22
|
+
{ columns: ['person_id', 'email_hash_v1'], primary: true },
|
|
23
|
+
{ columns: ['email_hash_v1'] },
|
|
24
|
+
{ columns: ['email_hash_md5'] }
|
|
25
|
+
]
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
name: 'person_hash_phone',
|
|
29
|
+
columns: {
|
|
30
|
+
person_id: 'person_id',
|
|
31
|
+
phone_hash_v1: {
|
|
32
|
+
type: 'hash',
|
|
33
|
+
description: 'SHA-256 hex of the normalized E.164-style phone (identity match key)'
|
|
34
|
+
},
|
|
35
|
+
phone_hash_md5: {
|
|
36
|
+
type: 'hash',
|
|
37
|
+
nullable: true,
|
|
38
|
+
default_value: '',
|
|
39
|
+
description: 'MD5 hex of the normalized phone; empty when plaintext was not available'
|
|
40
|
+
},
|
|
41
|
+
source_input_id: 'foreign_uuid',
|
|
42
|
+
created_at: 'created_at',
|
|
43
|
+
modified_at: 'modified_at'
|
|
44
|
+
},
|
|
45
|
+
indexes: [
|
|
46
|
+
{ columns: ['person_id', 'phone_hash_v1'], primary: true },
|
|
47
|
+
{ columns: ['phone_hash_v1'] },
|
|
48
|
+
{ columns: ['phone_hash_md5'] }
|
|
49
|
+
]
|
|
50
|
+
}
|
|
51
|
+
];
|
|
52
|
+
export default {
|
|
53
|
+
tables
|
|
54
|
+
};
|