@withtruing/companion 0.13.2 → 0.14.0

package/dist/whole-read/owner-decision-detectors.js

@@ -0,0 +1,270 @@
+export const LARGE_DATA_MINIMUM_BYTES = 256 * 1024;
+export const MINIFIED_MINIMUM_BYTES = 32 * 1024;
+const RULES_VERSION = 'inventory-rules@1';
+function parsesJson(text) {
+    try {
+        JSON.parse(text);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function parsesNdjson(text, lines) {
+    if (lines.length === 0)
+        return false;
+    return lines.every((line) => {
+        const value = text.slice(line.startCharacter, line.contentEndCharacter);
+        return value.trim().length > 0 && parsesJson(value);
+    });
+}
+function parsesDelimited(text, delimiter) {
+    let quoted = false;
+    let atFieldStart = true;
+    let sawDelimiter = false;
+    for (let index = 0; index < text.length; index += 1) {
+        const character = text[index];
+        if (quoted) {
+            if (character === '"') {
+                if (text[index + 1] === '"')
+                    index += 1;
+                else
+                    quoted = false;
+            }
+            continue;
+        }
+        if (character === '"') {
+            if (!atFieldStart)
+                return false;
+            quoted = true;
+            atFieldStart = false;
+        }
+        else if (character === delimiter) {
+            sawDelimiter = true;
+            atFieldStart = true;
+        }
+        else if (character === '\n') {
+            atFieldStart = true;
+        }
+        else if (character === '\r') {
+            if (text[index + 1] !== '\n')
+                return false;
+        }
+        else
+            atFieldStart = false;
+    }
+    return !quoted && sawDelimiter;
+}
+const XML_NAME = /^[A-Za-z_][A-Za-z0-9._:-]*/;
+function parsesXml(text) {
+    if (/<!DOCTYPE/i.test(text) || /<!ENTITY/i.test(text))
+        return false;
+    const tags = [];
+    let rootCount = 0;
+    let position = 0;
+    while (position < text.length) {
+        const next = text.indexOf('<', position);
+        const content = text.slice(position, next < 0 ? text.length : next);
+        if (/[&](?!amp;|lt;|gt;|apos;|quot;|#[0-9]+;|#x[0-9a-fA-F]+;)/.test(content)
+            || (tags.length === 0 && content.trim().length > 0))
+            return false;
+        if (next < 0)
+            return rootCount === 1 && tags.length === 0;
+        if (text.startsWith('<!--', next)) {
+            const end = text.indexOf('-->', next + 4);
+            if (end < 0 || text.slice(next + 4, end).includes('--'))
+                return false;
+            position = end + 3;
+            continue;
+        }
+        if (text.startsWith('<![CDATA[', next)) {
+            const end = text.indexOf(']]>', next + 9);
+            if (end < 0)
+                return false;
+            position = end + 3;
+            continue;
+        }
+        if (text.startsWith('<?', next)) {
+            const end = text.indexOf('?>', next + 2);
+            if (end < 0)
+                return false;
+            position = end + 2;
+            continue;
+        }
+        const close = text.indexOf('>', next + 1);
+        if (close < 0)
+            return false;
+        const token = text.slice(next + 1, close);
+        if (token.startsWith('/')) {
+            const name = token.slice(1).trim();
+            if (!XML_NAME.test(name) || XML_NAME.exec(name)[0] !== name || tags.pop() !== name)
+                return false;
+        }
+        else {
+            const body = token.endsWith('/') ? token.slice(0, -1).trimEnd() : token;
+            const match = XML_NAME.exec(body);
+            if (match === null)
+                return false;
+            const name = match[0];
+            const attributes = body.slice(name.length);
+            if (!/^(?:\s+[A-Za-z_][A-Za-z0-9._:-]*\s*=\s*(?:"[^"<]*"|'[^'<]*'))*\s*$/.test(attributes)
+                || /[&](?!amp;|lt;|gt;|apos;|quot;|#[0-9]+;|#x[0-9a-fA-F]+;)/.test(attributes))
+                return false;
+            if (tags.length === 0 && ++rootCount > 1)
+                return false;
+            if (!token.endsWith('/'))
+                tags.push(name);
+        }
+        position = close + 1;
+    }
+    return tags.length === 0;
+}
+function sqlStatements(text) {
+    const statements = [];
+    let start = 0;
+    let quote = null;
+    let lineComment = false;
+    let blockComment = false;
+    for (let index = 0; index < text.length; index += 1) {
+        const character = text[index];
+        const next = text[index + 1];
+        if (lineComment) {
+            if (character === '\n')
+                lineComment = false;
+            continue;
+        }
+        if (blockComment) {
+            if (character === '*' && next === '/') {
+                blockComment = false;
+                index += 1;
+            }
+            continue;
+        }
+        if (quote !== null) {
+            if (character === quote) {
+                if (next === quote)
+                    index += 1;
+                else
+                    quote = null;
+            }
+            continue;
+        }
+        if (character === '-' && next === '-') {
+            lineComment = true;
+            index += 1;
+            continue;
+        }
+        if (character === '/' && next === '*') {
+            blockComment = true;
+            index += 1;
+            continue;
+        }
+        if (character === "'" || character === '"') {
+            quote = character;
+            continue;
+        }
+        if (character === ';') {
+            statements.push(text.slice(start, index));
+            start = index + 1;
+        }
+    }
+    if (quote !== null || blockComment)
+        return null;
+    statements.push(text.slice(start));
+    return statements;
+}
+function stripSqlComments(statement) {
+    return statement.replace(/--[^\n]*(?:\n|$)|\/\*[\s\S]*?\*\//g, ' ').trim();
+}
+function parsesDataSql(text) {
+    const statements = sqlStatements(text);
+    if (statements === null)
+        return false;
+    let count = 0;
+    for (const statement of statements) {
+        const source = stripSqlComments(statement);
+        if (source.length === 0)
+            continue;
+        const command = /^([A-Za-z]+)/.exec(source)?.[1]?.toUpperCase();
+        if (command === 'INSERT') {
+            if (!/^INSERT\s+INTO\s+[^\s(]+(?:\s*\([^)]*\))?\s+VALUES\s*\(/is.test(source))
+                return false;
+        }
+        else if (command === 'COPY') {
+            if (!/^COPY\s+.+\s+(?:FROM|TO)\s+(?!PROGRAM\b)\S+/is.test(source))
+                return false;
+        }
+        else
+            return false;
+        count += 1;
+    }
+    return count > 0;
+}
+function startsWithSqlCommand(text) {
+    const statements = sqlStatements(text);
+    if (statements === null)
+        return false;
+    for (const statement of statements) {
+        const command = /^([A-Za-z]+)/.exec(stripSqlComments(statement))?.[1]?.toUpperCase();
+        if (command !== undefined)
+            return new Set(['ALTER', 'COPY', 'CREATE', 'DELETE', 'DROP', 'INSERT', 'SELECT', 'UPDATE']).has(command);
+    }
+    return false;
+}
+function dataFormat(text, lines) {
+    if (parsesJson(text))
+        return 'json';
+    if (parsesNdjson(text, lines))
+        return 'ndjson';
+    if (parsesXml(text))
+        return 'xml';
+    if (startsWithSqlCommand(text))
+        return parsesDataSql(text) ? 'sql' : null;
+    if (parsesDelimited(text, ','))
+        return 'csv';
+    if (parsesDelimited(text, '\t'))
+        return 'tsv';
+    return null;
+}
+/** Deterministic ISO-8859-1 decoding; this intentionally is not Windows-1252. */
+export function decodeLatin1(bytes) {
+    let decoded = '';
+    for (const byte of bytes)
+        decoded += String.fromCharCode(byte);
+    return decoded;
+}
+/** Returns possible owner decisions only. It neither changes disposition nor selects a choice. */
+export function detectOwnerDecisions(input) {
+    const canonicalBytes = input.canonicalBytes.byteLength;
+    if (input.baseClassification === 'unknown_format')
+        return [{
+                entryId: input.entryId, kind: 'unknown_format', choices: ['set aside as binary'], rulesVersion: RULES_VERSION,
+                detection: { canonicalBytes },
+            }];
+    if (input.baseClassification === 'unsupported_encoding')
+        return [{
+                entryId: input.entryId, kind: 'unsupported_encoding', choices: ['read as Latin-1', 'set aside'], rulesVersion: RULES_VERSION,
+                detection: { canonicalBytes },
+            }];
+    if (input.decoded.status !== 'decoded')
+        return [];
+    const candidates = [];
+    if (canonicalBytes >= LARGE_DATA_MINIMUM_BYTES) {
+        const format = dataFormat(input.decoded.text, input.decoded.mapping.lines);
+        if (format !== null)
+            candidates.push({
+                entryId: input.entryId, kind: 'large_data', choices: ['read', 'set aside as data'], rulesVersion: RULES_VERSION,
+                detection: { canonicalBytes, format },
+            });
+    }
+    if (!input.libraryMatchEstablished && canonicalBytes >= MINIFIED_MINIMUM_BYTES && input.decoded.mapping.lines.length > 0) {
+        const contentLength = input.decoded.mapping.lines.reduce((total, line) => total + line.contentEndCharacter - line.startCharacter, 0);
+        const meanLineLength = contentLength / input.decoded.mapping.lines.length;
+        if (meanLineLength >= 500)
+            candidates.push({
+                entryId: input.entryId, kind: 'minified', choices: ['read', 'set aside as build output'], rulesVersion: RULES_VERSION,
+                detection: { canonicalBytes, meanLineLength, lineCount: input.decoded.mapping.lines.length },
+            });
+    }
+    return candidates;
+}