firefly-compiler 0.6.20 → 0.6.24

package/lsp/Sql.ff

@@ -0,0 +1,1395 @@
+import Token from ff:compiler
+import Syntax from ff:compiler
+import Pg from ff:postgresql
+import Environment from ff:compiler
+import Sql
+import SqlStatement
+
+data SqlTokenKind {
+    SqlReserved
+    SqlUnreserved
+    SqlIdentifier
+    SqlNumber
+    SqlString
+    SqlOperator
+    SqlPunctuation
+    SqlComment
+    SqlPlaceholder
+}
+
+data SqlToken(
+    file: String
+    code: String
+    codeLower: String
+    kind: SqlTokenKind
+    startLine: Int
+    startLineOffset: Int
+    startOffset: Int
+    stopLine: Int
+    stopLineOffset: Int
+    stopOffset: Int
+)
+
+extend token: SqlToken {
+
+    at(): Location {
+        Location(token.file, token.startLine, (token.startOffset - token.startLineOffset) + 1)
+    }
+
+    end(): Location {
+        Location(token.file, token.stopLine, (token.stopOffset - token.stopLineOffset) + 1)
+    }
+
+    raw(): String {
+        token.code.slice(token.startOffset, token.stopOffset)
+    }
+
+    is(kind: SqlTokenKind): Bool {
+        token.kind == kind
+    }
+
+    is2(kind1: SqlTokenKind, kind2: SqlTokenKind): Bool {
+        token.kind == kind1 || token.kind == kind2
+    }
+
+    is3(kind1: SqlTokenKind, kind2: SqlTokenKind, kind3: SqlTokenKind): Bool {
+        token.kind == kind1 || token.kind == kind2 || token.kind == kind3
+    }
+
+    rawIs(value: String): Bool {
+        token.stopOffset - token.startOffset == value.size() &&
+        token.codeLower.startsWith(value, token.startOffset)
+    }
+}
+
+tokenize(sql: String): List[SqlToken] {
+    let sqlLower = sql.lower()
+    let tokens = Array.new()
+    mutable offset = 0
+    mutable line = 1
+    mutable lineOffset = 0
+    let length = sql.size()
+    function emitToken(startOffset: Int, kind: SqlTokenKind) {
+        tokens.push(SqlToken(".", sql, sqlLower, kind, line, lineOffset, startOffset, line, lineOffset, offset))
+    }
+    while {offset < length} {
+        let char = sql.grab(offset)
+        let startOffset = offset
+        if(char == '\n') {
+            offset += 1
+            line += 1
+            lineOffset = offset
+        } elseIf {char == '-' && offset + 1 < length && sql.grab(offset + 1) == '-'} {
+            offset += 2
+            while {offset < length && sql.grab(offset) != '\n'} {
+                offset += 1
+            }
+            emitToken(startOffset, SqlComment)
+        } elseIf {char == '/' && offset + 1 < length && sql.grab(offset + 1) == '*'} {
+            offset += 2
+            while {offset < length && !(sql.grab(offset) == '*' && offset + 1 < length && sql.grab(offset + 1) == '/')} {
+                if(sql.grab(offset) == '\n') {
+                    line += 1
+                    lineOffset = offset + 1
+                }
+                offset += 1
+            }
+            offset += 2
+            emitToken(startOffset, SqlComment)
+        } elseIf {char == '\''} {
+            offset += 1
+            while {offset < length && sql.grab(offset) != '\''} {
+                if(sql.grab(offset) == '\\' && offset + 1 < length) {
+                    offset += 2
+                } else {
+                    if(sql.grab(offset) == '\n') {
+                        line += 1
+                        lineOffset = offset + 1
+                    }
+                    offset += 1
+                }
+            }
+            offset += 1
+            emitToken(startOffset, SqlString)
+        } elseIf {char == '$'} {
+            offset += 1
+            while {offset < length && sql.grab(offset).isAsciiLetterOrDigit()} {
+                offset += 1
+            }
+            emitToken(startOffset, SqlPlaceholder)
+        } elseIf {char.isAsciiDigit()} {
+            while {offset < length && sql.grab(offset).isAsciiDigit()} {
+                offset += 1
+            }
+            if(offset < length && sql.grab(offset) == '.') {
+                offset += 1
+                while {offset < length && sql.grab(offset).isAsciiDigit()} {
+                    offset += 1
+                }
+            }
+            emitToken(startOffset, SqlNumber)
+        } elseIf {char.isAsciiLetter() || char == '_' || char == '"'} {
+            let isQuoted = char == '"'
+            if(isQuoted) {
+                offset += 1
+            }
+            while {offset < length && (
+                sql.grab(offset).isAsciiLetterOrDigit() || sql.grab(offset) == '_' || 
+                (isQuoted && sql.grab(offset) != '"')
+            )} {
+                offset += 1
+            }
+            if(isQuoted && offset < length && sql.grab(offset) == '"') {
+                offset += 1
+            }
+            let word = sqlLower.slice(startOffset, offset)
+            if(!isQuoted && sqlReserved.contains(word) && (tokens.isEmpty() || !tokens.grabLast().rawIs("as"))) {
+                emitToken(startOffset, SqlReserved)
+            } elseIf {
+                !isQuoted && sqlUnreserved.contains(word) && (tokens.isEmpty() || !tokens.grabLast().rawIs("as"))
+            } {
+                emitToken(startOffset, SqlUnreserved)
+            } else {
+                emitToken(startOffset, SqlIdentifier)
+            }
+        } elseIf {char == '(' || char == ')' || char == ',' || char == ';' || char == '.'} {
+            offset += 1
+            emitToken(startOffset, SqlPunctuation)
+        } elseIf {char == ':' && offset + 1 < length && sql.grab(offset + 1) == ':'} {
+            offset += 2
+            emitToken(startOffset, SqlPunctuation)
+        } else {
+            mutable operator = False
+            mutable c = char
+            while {(
+                c == '+' || c == '-' || c == '*' || c == '/' || c == '<' || c == '>' || c == '=' || 
+                c == '~' || c == '!' || c == '@' || c == '#' || c == '%' || c == '^' || c == '&' || 
+                c == '|' || c == '`' || c == '?'
+            ) && !(
+                c == '-' && offset + 1 < length && sql.grab(offset + 1) == '-'
+            ) && !(
+                c == '/' && offset + 1 < length && sql.grab(offset + 1) == '*'
+            )} {
+                operator = True
+                offset += 1
+                c = if(offset < length) {sql.grab(offset)} else {' '}
+            }
+            if(operator) {
+                emitToken(startOffset, SqlOperator)
+            } else {
+                offset += 1
+            }
+        }
+    }
+    tokens.drain()
+}
+
+class SqlScope(
+    mutable targetTable: Option[Pair[Option[String], String]]
+    symbols: Array[SqlSymbol]
+)
+
+data SqlSymbol {
+    SqlQuery(alias: String, columns: List[String])
+    SqlTable(alias: Option[String], schema: Option[String], table: String)
+}
+
+data SqlSimpleView(schema: Option[String], table: String, columns: List[String])
+
+data SqlCompletionScope(
+    targetTable: Option[Pair[Option[String], String]]
+    expectingColumnNames: Option[Set[String]] // Seen set
+    expectingColumnSetters: Option[Set[String]] // Seen set
+    expectingJoin: Bool
+    expectingTable: Bool
+    avoidTables: Bool
+    inQuotes: Bool
+    beforeDot: Option[String]
+    symbols: List[SqlSymbol]
+    columns: List[String]
+)
+
+completionScope(tokens: List[SqlToken], line: Int, column: Int): SqlCompletionScope {
+    let scopes = Array.new[SqlScope]()
+    mutable foundScopes = None
+    mutable offset = 0
+    mutable targetTable = None
+    mutable expectingColumnNames = None
+    mutable expectingColumnSetters = None
+    mutable expectingJoin = False
+    mutable expectingTable = False
+    mutable inQuotes = False
+    mutable avoidTables = False
+    mutable beforeDot = None
+    function rawIs(o: Int, raw: String): Bool {
+        o < tokens.size() && tokens.grab(o).rawIs(raw)
+    }
+    function is(o: Int, kind: SqlTokenKind): Bool {
+        o < tokens.size() && tokens.grab(o).is(kind)
+    }
+    function selectEnd(o: Int): Bool {
+        rawIs(o, "from") || rawIs(o, "join") || 
+        rawIs(o, "where") || rawIs(o, "group") || 
+        rawIs(o, "having") || rawIs(o, "window") ||
+        rawIs(o, "union") || rawIs(o, "intersect") || rawIs(o, "except") ||
+        rawIs(o, "order") || rawIs(o, "limit") || rawIs(o, "offset") ||
+        rawIs(o, "fetch") || rawIs(o, "for") || 
+        rawIs(o, "returning") || rawIs(o, ")") ||
+        (rawIs(o, "when") && (rawIs(o + 1, "matched") || rawIs(o + 2, "matched")))
+    }
+    function foundCursor(
+        expectTable: Bool
+        expectColumnNames: Option[Set[String]]
+        expectColumnSetters: Option[Set[String]]
+    ): Unit {
+        targetTable = scopes.last().flatMap {_.targetTable}
+        expectingColumnNames = expectColumnNames
+        expectingColumnSetters = expectColumnSetters
+        expectingJoin = offset >= 3 && (rawIs(offset - 1, "join") || (
+            rawIs(offset - 3, "join") && (is(offset - 2, SqlIdentifier) || is(offset - 2, SqlUnreserved)) &&
+            rawIs(offset - 1, ".")
+        ))
+        inQuotes = offset < tokens.size() && tokens.grab(offset).raw().startsWith("\"")
+        expectingTable = expectTable
+        beforeDot = if(offset < tokens.size() && offset >= 1 && rawIs(offset, ".") && (
+            is(offset - 1, SqlIdentifier) || is(offset - 1, SqlUnreserved)
+        )) {
+            let token = tokens.grab(offset)
+            if(token.stopLine == line && token.stopOffset - token.stopLineOffset >= column - 1) {
+                tokens.grab(offset - 1).raw()
+            }
+        } elseIf {offset >= 2 && rawIs(offset - 1, ".") && (
+            is(offset - 2, SqlIdentifier) || is(offset - 2, SqlUnreserved)
+        )} {Some(tokens.grab(offset - 2).raw())} else {
+            None
+        }.map {fromRaw(_)}
+        avoidTables = offset - 1 >= 0 && (
+            is(offset - 1, SqlIdentifier) || is(offset - 1, SqlUnreserved) || rawIs(offset - 1, ")")
+        )
+        foundScopes = Some(scopes.toList())
+    }
+    function checkCursor(
+        expectTable: Bool, 
+        expectColumnNames: Option[Set[String]] = None
+        expectColumnSetters: Option[Set[String]] = None
+    ): Unit {
+        if(!foundScopes.isEmpty()) {} else:
+        if(offset >= tokens.size()) {foundCursor(expectTable, expectColumnNames, expectColumnSetters)} else:
+        let token = tokens.grab(offset)
+        let grace = if(token.is(SqlPunctuation)) {0} else {1}
+        if(token.stopLine > line || (
+            token.stopLine == line && token.stopOffset - token.stopLineOffset >= column - grace
+        )) {foundCursor(expectTable, expectColumnNames, expectColumnSetters)}
+    }
+    function parseScope(): List[Pair[String, Bool]] {
+        let scopeColumns = Array.new()
+        function parseColumns() {
+            mutable simple = True
+            while {offset < tokens.size() && !selectEnd(offset)} {
+                checkCursor(False)
+                if(rawIs(offset, "(")) {
+                    offset += 1
+                    parseScope()
+                    if(rawIs(offset, ")")) {offset += 1}
+                    simple = False
+                } elseIf {
+                    (is(offset, SqlIdentifier) || is(offset, SqlUnreserved)) &&
+                    (rawIs(offset + 1, ",") || selectEnd(offset + 1))
+                } {
+                    let alias = tokens.grab(offset)
+                    scopeColumns.push(Pair(fromRaw(alias.raw()), simple))
+                    simple = True
+                    offset += 1
+                    if(rawIs(offset, ",")) {
+                        checkCursor(False)
+                        offset += 1
+                    }
+                } else {
+                    simple = False
+                    offset += 1
+                }
+            }
+        }
+        function parseColumnNames(): Set[String] {
+            mutable seen = Set.new()
+            checkCursor(False, expectColumnNames = Some(seen))
+            while {offset < tokens.size() && !rawIs(offset, ")")} {
+                if(is(offset, SqlIdentifier) || is(offset, SqlUnreserved)) {
+                    offset += 1
+                    checkCursor(False, expectColumnNames = Some(seen))
+                    seen = seen.add(fromRaw(tokens.grab(offset - 1).raw()))
+                }
+                while {offset < tokens.size() && !rawIs(offset, ")") && !rawIs(offset, ",")} {
+                    offset += 1
+                    checkCursor(False)
+                }
+                if(rawIs(offset, ",")) {
+                    offset += 1
+                }
+                checkCursor(False, expectColumnNames = Some(seen))
+            }
+            seen
+        }
+        function parseColumnSetters() {
+            mutable seen = Set.new()
+            mutable beforeEquals = True
+            while {offset < tokens.size() && !selectEnd(offset)} {
+                checkCursor(False, expectColumnSetters = if(beforeEquals) {seen})
+                if(rawIs(offset, "(")) {
+                    offset += 1
+                    if(beforeEquals) {
+                        seen = seen.addAll(parseColumnNames())
+                    } else {
+                        parseScope()
+                    }
+                    if(rawIs(offset, ")")) {offset += 1}
+                } elseIf {rawIs(offset, ",")} {
+                    offset += 1
+                    beforeEquals = True
+                } elseIf {rawIs(offset, "=")} {
+                    offset += 1
+                    beforeEquals = False
+                } elseIf {beforeEquals && (is(offset, SqlIdentifier) || is(offset, SqlUnreserved))} {
+                    offset += 1
+                    checkCursor(False, expectColumnSetters = if(beforeEquals) {seen})
+                    seen = seen.add(fromRaw(tokens.grab(offset - 1).raw()))
+                    beforeEquals = False
+                } else {
+                    offset += 1
+                }
+            }
+            checkCursor(False, expectColumnSetters = if(beforeEquals) {seen})
+        }
+        let scope = SqlScope(None, Array.new())
+        scopes.push(scope)
+        if(rawIs(offset, "explain")) {offset += 1}
+        if(rawIs(offset, "analyze") || rawIs(offset, "analyse")) {offset += 1}
+        while {rawIs(offset, "with") || rawIs(offset, ",")} {
+            offset += 1
+            checkCursor(False)
+            if(is(offset, SqlIdentifier) || is(offset, SqlUnreserved)):
+            let alias = tokens.grab(offset)
+            offset += 1
+            if(rawIs(offset, "recursive")) {
+                offset += 1
+                if(rawIs(offset, "(")) {
+                    offset += 1
+                    parseScope()
+                    if(rawIs(offset, ")")) {offset += 1}
+                }
+            }
+            if(rawIs(offset, "as")) {offset += 1}
+            let columns = if(rawIs(offset, "(")) {
+                offset += 1
+                let columns = parseScope()
+                if(rawIs(offset, ")")) {offset += 1}
+                columns
+            } else {[]}
+            scope.symbols.push(SqlQuery(fromRaw(alias.raw()), columns.map {_.first}))
+        }
+        checkCursor(False)
+        mutable mergeUsing = rawIs(offset, "merge")
+        if(rawIs(offset, "select")) {
+            offset += 1
+            parseColumns()
+        }
+        while {
+            offset < tokens.size() && !rawIs(offset, ")") && !rawIs(offset, "returning") && 
+            !rawIs(offset, "union") && !rawIs(offset, "intersect") && !rawIs(offset, "except")
+        } {
+            checkCursor(False)
+            if(rawIs(offset, "(")) {
+                offset += 1
+                parseScope()
+                if(rawIs(offset, ")")) {offset += 1}
+            } elseIf {!scope.targetTable.isEmpty() && rawIs(offset, "insert") && rawIs(offset + 1, "(")} {
+                offset += 2
+                parseColumnNames()
+                if(rawIs(offset, ")")) {
+                    offset += 1
+                }
+            } elseIf {!scope.targetTable.isEmpty() && rawIs(offset, "update") && rawIs(offset + 1, "set")} {
+                offset += 2
+                parseColumnSetters()
+            } elseIf {
+                rawIs(offset, "from") || rawIs(offset, "join") || 
+                rawIs(offset, "update") || rawIs(offset, "into") ||
+                rawIs(offset, "only") || rawIs(offset, "table") || 
+                (mergeUsing && rawIs(offset, "using"))
+            } {
+                if(rawIs(offset, "using")) {mergeUsing = False}
+                let isFrom = rawIs(offset, "from")
+                let requireAs = rawIs(offset, "update") || rawIs(offset, "into") || rawIs(offset, "only")
+                doWhile {
+                    offset += 1
+                    if(rawIs(offset, "lateral")) {offset += 1}
+                    checkCursor(True)
+                    if(rawIs(offset, "(")) {
+                        offset += 1
+                        let columns = parseScope()
+                        if(rawIs(offset, ")")) {offset += 1}
+                        if(rawIs(offset, "as")) {offset += 1}
+                        if(is(offset, SqlIdentifier) || is(offset, SqlUnreserved)) {
+                            let alias = tokens.grab(offset)
+                            scope.symbols.push(SqlQuery(fromRaw(alias.raw()), columns.map {_.first}))
+                        }
+                    } elseIf {is(offset, SqlIdentifier) || is(offset, SqlUnreserved)} {
+                        let isTarget = offset - 1 >= 0 && (
+                            rawIs(offset - 1, "into") || rawIs(offset - 1, "update") || rawIs(offset - 1, "only")
+                        )
+                        let schema = if(rawIs(offset + 1, ".")) {
+                            let schema = tokens.grab(offset)
+                            offset += 2
+                            schema
+                        }
+                        checkCursor(True)
+                        if(is(offset, SqlIdentifier) || is(offset, SqlUnreserved)) {
+                            let relation = tokens.grab(offset)
+                            offset += 1
+                            let allowAlias = if(rawIs(offset, "as")) {offset += 1; True} else {!requireAs}
+                            checkCursor(True)
+                            let alias = if(allowAlias && (is(offset, SqlIdentifier) || is(offset, SqlUnreserved))) {
+                                let alias = tokens.grab(offset)
+                                offset += 1
+                                alias
+                            }
+                            if(isTarget) {
+                                scope.targetTable = Some(Pair(schema.map {fromRaw(_.raw())}, fromRaw(relation.raw())))
+                            }
+                            scope.symbols.push(SqlTable(
+                                alias.map {fromRaw(_.raw())}, schema.map {fromRaw(_.raw())}, fromRaw(relation.raw())
+                            ))
+                        }
+                        if(isTarget && rawIs(offset, "(")) {
+                            offset += 1
+                            parseColumnNames()
+                            if(rawIs(offset, ")")) {
+                                offset += 1
+                            }
+                        } elseIf {isTarget && rawIs(offset, "set")} {
+                            offset += 1
+                            parseColumnSetters()
+                        }
+                    }
+                    isFrom && rawIs(offset, ",")
+                }
+            } else {
+                offset += 1
+            }
+        }
+        checkCursor(False)
+        if(rawIs(offset, "returning")) {
+            offset += 1
+            parseColumns()
+        }
+        checkCursor(False)
+        scopes.pop()
+        while {rawIs(offset, "union") || rawIs(offset, "intersect") || rawIs(offset, "except")} {
+            offset += 1
+            checkCursor(False)
+            parseScope()
+        }
+        scopeColumns.drain()
+    }
+    let columns = parseScope().filter {_.second}.map {_.first}
+    let symbols = foundScopes.toList().flatten().flatMap {_.symbols.toList()}
+    SqlCompletionScope(
+        targetTable
+        expectingColumnNames
+        expectingColumnSetters
+        expectingJoin
+        expectingTable
+        avoidTables
+        inQuotes
+        beforeDot
+        symbols
+        columns
+    )
+}
+
+data SqlData(
+    searchPathSchemas: List[String]
+    relations: List[SqlRelation]
+    foreignKeys: List[SqlForeignKey]
+    functions: List[SqlFunction]
+)
+
+data SqlRelation(
+    kind: String 
+    schema: String
+    name: String
+    columns: List[SqlColumn]
+    simpleView: Option[SqlSimpleView]
+)
+
+data SqlColumn(
+    name: String
+    notNull: Bool
+    primaryKey: Bool
+    type: String
+    default: Option[String]
+    description: Option[String]
+)
+
+data SqlFunction(
+    kind: String
+    schema: String
+    name: String
+    argumentNames: List[String]
+    argumentTypes: List[String]
+    returnType: String
+    description: Option[String]
+)
+
+data SqlForeignKey(
+    schema: String
+    name: String
+    foreignSchema: String
+    foreignName: String
+    columns: List[Pair[String, String]]
+)
+
+extend self: SqlForeignKey {
+    swap(): SqlForeignKey {
+        SqlForeignKey(self.foreignSchema, self.foreignName, self.schema, self.name, self.columns.map {_.swap()})
+    }
+}
+
+extend self: SqlRelation {
+    tableLike(): Bool {
+        self.kind == "r" || self.kind == "v" || self.kind == "m" || self.kind == "f" || self.kind == "p"
+    }
+    kindName(): String {
+        self.kind.{
+            | "r" => "table"
+            | "v" => "view"
+            | "m" => "materialized view"
+            | "f" => "foreign table"
+            | "p" => "partitioned table"
+            | "i" => "index"
+            | "I" => "partitioned index"
+            | "S" => "sequence"
+            | "t" => "toast table"
+            | "c" => "composite type"
+            | _ => "other"
+        }
+    }
+}
+
+load(nodeSystem: NodeSystem, pool: PgPool): SqlData {
+    pool.transaction {connection =>
+        let s = loadSearchPath(connection)
+        let rs = loadRelations(connection)
+        let fs = loadFunctions(connection)
+        let ks = loadForeignKeys(connection)
+        SqlData(s, rs, ks, fs)
+    }
+}
+
+loadSearchPath(connection: PgConnection): List[String] {
+    connection.statement("""
+        select pg_catalog.current_schemas(true) search_path_schemas
+    """).map {row =>
+        row.getStringArray("search_path_schemas").grab()
+    }.grabFirst()
+}
+
+loadForeignKeys(connection: PgConnection): List[SqlForeignKey] {
+    connection.statement("""
+        select
+            n1.nspname schema_name,
+            cl1.relname table_name,
+            array_agg(a1.attname) column_names,
+            n2.nspname foreign_schema_name,
+            cl2.relname foreign_table_name,
+            array_agg(a2.attname) foreign_column_names
+        from pg_constraint c
+        join pg_attribute a1 on a1.attrelid = c.conrelid and a1.attnum = any(c.conkey)
+        join pg_attribute a2 on a2.attrelid = c.confrelid and a2.attnum = any(c.confkey)
+        join pg_class cl1 on cl1.oid = c.conrelid
+        join pg_class cl2 on cl2.oid = c.confrelid
+        join pg_namespace n1 on n1.oid = cl1.relnamespace
+        join pg_namespace n2 on n2.oid = cl2.relnamespace
+        where c.contype = 'f'
+        group by n1.nspname, cl1.relname, n2.nspname, cl2.relname, c.conname
+    """).map {row =>
+        let schema = row.getString("schema_name").grab()
+        let table = row.getString("table_name").grab()
+        let foreignSchema = row.getString("foreign_schema_name").grab()
+        let foreignTable = row.getString("foreign_table_name").grab()
+        let columns = row.getStringArray("column_names").else {[]}
+        let foreignColumns = row.getStringArray("foreign_column_names").else {[]}
+        SqlForeignKey(
+            schema = unquote(schema)
+            name = unquote(table)
+            foreignSchema = unquote(foreignSchema)
+            foreignName = unquote(foreignTable)
+            columns = columns.map {unquote(_)}.zip(foreignColumns.map {unquote(_)})
+        )
+    }
+}
+
+loadFunctions(connection: PgConnection): List[SqlFunction] {
+    connection.statement("""
+        select
+            p.prokind kind,
+            n.nspname schema_name,
+            p.proname function_name,
+            p.proargnames argument_names,
+            (
+                select pg_catalog.array_agg(pg_catalog.format_type(t, null)) 
+                from pg_catalog.unnest(p.proargtypes) t
+            ) argument_types,
+            pg_catalog.format_type(p.prorettype, null) return_type,
+            d.description function_description
+        from pg_catalog.pg_proc p
+        join pg_catalog.pg_namespace n on n.oid = p.pronamespace
+        left join pg_catalog.pg_description d on d.objoid = p.oid and d.classoid = 'pg_proc'::regclass
+        order by n.nspname, p.proname
+    """).map {row =>
+        SqlFunction(
+            kind = row.getString("kind").grab().{
+                | "p" => "procedure"
+                | "a" => "aggregate"
+                | "w" => "window"
+                | _ => "function"
+            }
+            schema = row.getString("schema_name").grab()
+            name = row.getString("function_name").grab()
+            argumentNames = row.getStringArray("argument_names").else {[]}.map {unquote(_)}
+            argumentTypes = row.getStringArray("argument_types").else {[]}.map {unquote(shortenType(_))}
+            returnType = unquote(shortenType(row.getString("return_type").grab()))
+            description = row.getString("function_description")
+        )
+    }
+}
+
+loadRelations(connection: PgConnection): List[SqlRelation] {
+    let results = connection.statement("""
+        select
+            c.relkind kind,
+            n.nspname schema_name,
+            c.relname relation_name,
+            a.attname column_name,
+            a.attnotnull not_null,
+            pg_catalog.format_type(a.atttypid, a.atttypmod) data_type,
+            pg_catalog.col_description(c.oid, a.attnum) column_description,
+            exists (
+                select 1 from pg_catalog.pg_index i
+                where i.indrelid = a.attrelid and i.indisprimary and a.attnum = any(i.indkey)
+            ) primary_key,
+            pg_catalog.pg_get_expr(d.adbin, d.adrelid) default_value,
+            case when c.relkind = 'v' and a.attnum = 1 then pg_catalog.pg_get_viewdef(c.oid) end view_definition
+        from pg_catalog.pg_class c
+        join pg_catalog.pg_namespace n on n.oid = c.relnamespace
+        left join pg_catalog.pg_attribute a on a.attrelid = c.oid and a.attnum > 0 and not a.attisdropped
+        left join pg_catalog.pg_attrdef d on d.adrelid = a.attrelid and d.adnum = a.attnum and c.relkind = 'r'
+        order by n.nspname, c.relname, a.attnum
+    """).map {row =>
+        let schemaName = unquote(row.getString("schema_name").grab())
+        let tableName = unquote(row.getString("relation_name").grab())
+        let simpleView = row.getString("view_definition").flatMap {d =>
+            let tokens = tokenize(d)
+            let scope = completionScope(tokens, 1, 1)
+            scope.symbols.{
+                | [SqlTable(_, schema, table)] => Some(SqlSimpleView(schema, table, scope.columns))
+                | _ => None
+            }
+        }
+        SqlRelation(
+            kind = row.getString("kind").grab()
+            schema = schemaName
+            name = tableName
+            columns = row.getString("column_name").toList().map {unquote(_)}.map {columnName =>
+                let type = unquote(shortenType(row.getString("data_type").grab()))
+                let default = row.getString("default_value").map {d =>
+                    tokenize(d).{
+                        | [a, b, c] {a.is(SqlString) && b.rawIs("::") && c.rawIs("text")} =>
+                            a.raw()
+                        | _ => 
+                            d
+                    }
+                }
+                let serialType = if(!default.any {_.endsWith("::regclass)")}) {type} else {
+                    if(!default.map {_.replace("\"", "")}.any {d => 
+                        d == "nextval('" + tableName + "_" + columnName + "_seq'::regclass)" || 
+                        d == "nextval('" + schemaName + "." + tableName + "_" + columnName + "_seq'::regclass)"
+                    }) {type} else:
+                    if(type == "bigint") {"bigserial"} else:
+                    if(type == "integer") {"serial"} else:
+                    type
+                }
+                SqlColumn(
+                    name = columnName
+                    notNull = row.getBool("not_null").grab()
+                    primaryKey = row.getBool("primary_key").grab()
+                    type = serialType
+                    default = default.filter {_ => type == serialType}
+                    description = row.getString("column_description")
+                )
+            }
+            simpleView = simpleView
+        )
+    }
+    let groups = results.map {r => Pair(Pair(r.schema, r.name), r)}.group().toList()
+    let relations = groups.map {| Pair(_, rs) =>
+        rs.grabFirst().SqlRelation(
+            columns = rs.flatMap {_.columns}.sortBy {c => Pair(if(c.primaryKey) {1} else {2}, c.name)}
+        )
+    }.sortBy {r => Pair(r.schema, r.name)}
+    let tables = relations.filter {_.kind == "r"}.map {r => Pair(Pair(r.schema, r.name), r)}.toMap()
+    relations.map {r =>
+        r.simpleView.flatMap {v =>
+            v.schema.flatMap {tables.get(Pair(_, v.table))}.map {t =>
+                let simpleColumns = v.columns.toSet()
+                r.SqlRelation(columns = r.columns.map {c =>
+                    if(!simpleColumns.contains(c.name)) {c} else {
+                        t.columns.find {c2 => c2.name == c.name}.else {c}
+                    }
+                })
+            }
+        }.else {r}
+    }
+}
+
+data ExplainedStatement(
+    explainStatement: String
+    deltas: List[Pair[Int, Int]]
+    usedParameters: List[Pair[String, Int]]
+)
+
+buildExplainStatement(statement: String, parameters: Option[List[SqlParameter]]): Option[ExplainedStatement] {
+    let tokens = tokenize(statement).filter {_.kind != SqlComment}
+    let placeholders = tokens.filter {token => 
+        token.kind.{
+            | SqlPlaceholder => True
+            | _ => False
+        }
+    }
+    let replacements = placeholders.map {p => Pair(p.raw(), p)}.group().toList().pairs().flatMap {
+        | Pair(index, Pair(raw, tokens)) => tokens.map {token => Pair(token.startOffset, Pair(index, raw))}
+    }
+    let prefix = "EXPLAIN (GENERIC_PLAN) "
+    mutable fixedStatement = prefix
+    let deltas = [Pair(0, -prefix.size())].toArray()
+    mutable position = 0
+    let usedParameters = replacements.map {| Pair(offset, Pair(index, raw)) => 
+        Pair(raw.dropFirst(), offset)
+    }
+    let types = parameters.toList().flatten().map {p => Pair(p.name, p.sqlType)}.toMap()
+    replacements.each {| Pair(offset, Pair(index, raw)) => 
+        let simplePlaceholder = "$" + (index + 1)
+        let placeholder = types.get(raw.dropFirst()).map {sqlType => 
+            "(" + simplePlaceholder + "::" + sqlType + ")"
+        }.else {simplePlaceholder}
+        let part = statement.slice(position, offset)
+        fixedStatement += part + placeholder
+        position += part.size() + raw.size()
+        deltas.push(Pair(fixedStatement.size() - placeholder.size(), raw.size() - placeholder.size()))
+    }
+    fixedStatement += statement.slice(position, statement.size())
+    let noInbetweenSemicolon = tokens.indexWhere {_.rawIs(";")}.all {_ >= tokens.size() - 1}
+    if(noInbetweenSemicolon && tokens.first().any {t => 
+        t.rawIs("with") || t.rawIs("select") || t.rawIs("values") || 
+        t.rawIs("insert") || t.rawIs("update") || t.rawIs("delete") || t.rawIs("merge")
+    }):
+    ExplainedStatement(fixedStatement, deltas.drain(), usedParameters)
+}
+
+shortenType(fullType: String): String {
+    fullType
+        .replace("timestamp with time zone", "timestamptz")
+        .replace("timestamp without time zone", "timestamp")
+        .replace("time without time zone", "time")
+        .replace("double precision", "float")
+}
+
+quote(identifier: String, force: Bool = False): String {
+    if(!force && identifier.first().any {_.isAsciiLower()} && identifier.all {c => 
+        c.isAsciiLower() || c.isAsciiDigit() || c == '_'
+    }) {identifier} else {
+        "\"" + identifier.replace("\"", "\"\"") + "\""
+    }
+}
+
+unquote(identifier: String): String {
+    if(identifier.startsWith("\"")) {
+        identifier.dropFirst().dropLast().replace("\"\"", "\"")
+    } else {
+        identifier
+    }
+}
+
+fromRaw(identifier: String): String {
+    if(identifier.startsWith("\"")) {
+        identifier.dropFirst().dropLast().replace("\"\"", "\"")
+    } else {
+        identifier.lower()
+    }
+}
+
+// Source: https://github.com/postgres/postgres/blob/master/src/include/parser/kwlist.h
+sqlReserved: Set[String] = [
+    "all"
+    "analyse"
+    "analyze"
+    "and"
+    "any"
+    "array"
+    "as"
+    "asc"
+    "asymmetric"
+    "authorization"
+    "between"
+    "bigint"
+    "binary"
+    "bit"
+    "boolean"
+    "both"
+    "case"
+    "cast"
+    "char"
+    "character"
+    "check"
+    "coalesce"
+    "collate"
+    "collation"
+    "column"
+    "concurrently"
+    "constraint"
+    "create"
+    "cross"
+    "current_catalog"
+    "current_date"
+    "current_role"
+    "current_schema"
+    "current_time"
+    "current_timestamp"
+    "current_user"
+    "dec"
+    "decimal"
+    "default"
+    "deferrable"
+    "desc"
+    "distinct"
+    "do"
+    "else"
+    "end"
+    "except"
+    "exists"
+    "extract"
+    "false"
+    "fetch"
+    "float"
+    "for"
+    "foreign"
+    "freeze"
+    "from"
+    "full"
+    "grant"
+    "graph_table"
+    "greatest"
+    "group"
+    "grouping"
+    "having"
+    "ilike"
+    "in"
+    "initially"
+    "inner"
+    "inout"
+    "int"
+    "integer"
+    "intersect"
+    "interval"
+    "into"
+    "is"
+    "isnull"
+    "join"
+    "json"
+    "json_array"
+    "json_arrayagg"
+    "json_exists"
+    "json_object"
+    "json_objectagg"
+    "json_query"
+    "json_scalar"
+    "json_serialize"
+    "json_table"
+    "json_value"
+    "lateral"
+    "leading"
+    "least"
+    "left"
+    "like"
+    "limit"
+    "localtime"
+    "localtimestamp"
+    "merge_action"
+    "national"
+    "natural"
+    "nchar"
+    "none"
+    "normalize"
+    "not"
+    "notnull"
+    "null"
+    "nullif"
+    "numeric"
+    "offset"
+    "on"
+    "only"
+    "or"
+    "order"
+    "out"
+    "outer"
+    "overlaps"
+    "overlay"
+    "placing"
+    "position"
+    "precision"
+    "primary"
+    "real"
+    "references"
+    "returning"
+    "right"
+    "row"
+    "select"
+    "session_user"
+    "setof"
+    "similar"
+    "smallint"
+    "some"
+    "substring"
+    "symmetric"
+    "system_user"
+    "table"
+    "tablesample"
+    "then"
+    "time"
+    "timestamp"
+    "to"
+    "trailing"
+    "treat"
+    "trim"
+    "true"
+    "union"
+    "unique"
+    "user"
+    "using"
+    "values"
+    "varchar"
+    "variadic"
+    "verbose"
+    "when"
+    "where"
+    "window"
+    "with"
+    "xmlattributes"
+    "xmlconcat"
+    "xmlelement"
+    "xmlexists"
+    "xmlforest"
+    "xmlnamespaces"
+    "xmlparse"
+    "xmlpi"
+    "xmlroot"
+    "xmlserialize"
+    "xmltable"
+].toSet()
+
+sqlUnreserved: Set[String] = [
+    "abort"
+    "absent"
+    "absolute"
+    "access"
+    "action"
+    "add"
+    "admin"
+    "after"
+    "aggregate"
+    "also"
+    "alter"
+    "always"
+    "asensitive"
+    "assertion"
+    "assignment"
+    "at"
+    "atomic"
+    "attach"
+    "attribute"
+    "backward"
+    "before"
+    "begin"
+    "breadth"
+    "by"
+    "cache"
+    "call"
+    "called"
+    "cascade"
+    "cascaded"
+    "catalog"
+    "chain"
+    "characteristics"
+    "checkpoint"
+    "class"
+    "close"
+    "cluster"
+    "columns"
+    "comment"
+    "comments"
+    "commit"
+    "committed"
+    "compression"
+    "conditional"
+    "configuration"
+    "conflict"
+    "connection"
+    "constraints"
+    "content"
+    "continue"
+    "conversion"
+    "copy"
+    "cost"
+    "csv"
+    "cube"
+    "current"
+    "cursor"
+    "cycle"
+    "data"
+    "database"
+    "day"
+    "deallocate"
+    "declare"
+    "defaults"
+    "deferred"
+    "definer"
+    "delete"
+    "delimiter"
+    "delimiters"
+    "depends"
+    "depth"
+    "destination"
+    "detach"
+    "dictionary"
+    "disable"
+    "discard"
+    "document"
+    "domain"
+    "double"
+    "drop"
+    "each"
+    "edge"
+    "empty"
+    "enable"
+    "encoding"
+    "encrypted"
+    "enforced"
+    "enum"
+    "error"
+    "escape"
+    "event"
+    "exclude"
+    "excluding"
+    "exclusive"
+    "execute"
+    "explain"
+    "expression"
+    "extension"
+    "external"
+    "family"
+    "filter"
+    "finalize"
+    "first"
+    "following"
+    "force"
+    "format"
+    "forward"
+    "function"
+    "functions"
+    "generated"
+    "global"
+    "granted"
+    "graph"
+    "groups"
+    "handler"
+    "header"
+    "hold"
+    "hour"
+    "identity"
+    "if"
+    "ignore"
+    "immediate"
+    "immutable"
+    "implicit"
+    "import"
+    "include"
+    "including"
+    "increment"
+    "indent"
+    "index"
+    "indexes"
+    "inherit"
+    "inherits"
+    "inline"
+    "input"
+    "insensitive"
+    "insert"
+    "instead"
+    "invoker"
+    "isolation"
+    "keep"
+    "key"
+    "keys"
+    "label"
+    "language"
+    "large"
+    "last"
+    "leakproof"
+    "level"
+    "listen"
+    "load"
+    "local"
+    "location"
+    "lock"
+    "locked"
+    "logged"
+    "lsn"
+    "mapping"
+    "match"
+    "matched"
+    "materialized"
+    "maxvalue"
+    "merge"
+    "method"
+    "minute"
+    "minvalue"
+    "mode"
+    "month"
+    "move"
+    "name"
+    "names"
+    "nested"
+    "new"
+    "next"
+    "nfc"
+    "nfd"
+    "nfkc"
+    "nfkd"
+    "no"
+    "node"
+    "normalized"
+    "nothing"
+    "notify"
+    "nowait"
+    "nulls"
+    "object"
+    "objects"
+    "of"
+    "off"
+    "oids"
+    "old"
+    "omit"
+    "operator"
+    "option"
+    "options"
+    "ordinality"
+    "others"
+    "over"
+    "overriding"
+    "owned"
+    "owner"
+    "parallel"
+    "parameter"
+    "parser"
+    "partial"
+    "partition"
+    "partitions"
+    "passing"
+    "password"
+    "path"
+    "period"
+    "plan"
+    "plans"
+    "policy"
+    "portion"
+    "preceding"
+    "prepare"
+    "prepared"
+    "preserve"
+    "prior"
+    "privileges"
+    "procedural"
+    "procedure"
+    "procedures"
+    "program"
+    "properties"
+    "property"
+    "publication"
+    "quote"
+    "quotes"
+    "range"
+    "read"
+    "reassign"
+    "recursive"
+    "ref"
+    "referencing"
+    "refresh"
+    "reindex"
+    "relationship"
+    "relative"
+    "release"
+    "rename"
+    "repack"
+    "repeatable"
+    "replace"
+    "replica"
+    "reset"
+    "respect"
+    "restart"
+    "restrict"
+    "return"
+    "returns"
+    "revoke"
+    "role"
+    "rollback"
+    "rollup"
+    "routine"
+    "routines"
+    "rows"
+    "rule"
+    "savepoint"
+    "scalar"
+    "schema"
+    "schemas"
+    "scroll"
+    "search"
+    "second"
+    "security"
+    "sequence"
+    "sequences"
+    "serializable"
+    "server"
+    "session"
+    "set"
+    "sets"
+    "share"
+    "show"
+    "simple"
+    "skip"
+    "snapshot"
+    "source"
+    "split"
+    "sql"
+    "stable"
+    "standalone"
+    "start"
+    "statement"
+    "statistics"
+    "stdin"
+    "stdout"
+    "storage"
+    "stored"
+    "strict"
+    "string"
+    "strip"
+    "subscription"
+    "support"
+    "sysid"
+    "system"
+    "tables"
+    "tablespace"
+    "target"
+    "temp"
+    "template"
+    "temporary"
+    "text"
+    "ties"
+    "transaction"
+    "transform"
+    "trigger"
+    "truncate"
+    "trusted"
+    "type"
+    "types"
+    "uescape"
+    "unbounded"
+    "uncommitted"
+    "unconditional"
+    "unencrypted"
+    "unknown"
+    "unlisten"
+    "unlogged"
+    "until"
+    "update"
+    "vacuum"
+    "valid"
+    "validate"
+    "validator"
+    "value"
+    "varying"
+    "version"
+    "vertex"
+    "view"
+    "views"
+    "virtual"
+    "volatile"
+    "wait"
+    "whitespace"
+    "within"
+    "without"
+    "work"
+    "wrapper"
+    "write"
+    "xml"
+    "year"
+    "yes"
+    "zone"
+].toSet()
+
+nodeMain(system: NodeSystem) {
+    function showToken(token: SqlToken): String {
+        "Token(" + Show.show(token.kind) + ", \"" + token.raw() + "\", at: " + 
+        token.at().line + ":" + token.at().column + ")"
+    }
+    function showCompletionScope(sql: String, find: String) {
+        system.writeLine("")
+        let foundIndex = sql.indexOf(find).grab()
+        let scope = completionScope(tokenize(sql), 1, 1 + foundIndex)
+        system.writeLine(sql)
+        system.writeLine(
+            " ".repeat(foundIndex) + "^ " + 
+            if(!scope.targetTable.isEmpty()) {
+                let table = scope.targetTable.grab()
+                table.first.map {_ + "."}.else {""} + table.second +
+                if(!scope.expectingColumnNames.isEmpty()) {
+                    " (" + scope.expectingColumnNames.grab().toList().join(", ") + ")"
+                } elseIf {!scope.expectingColumnSetters.isEmpty()} {
+                    " SET " + scope.expectingColumnSetters.grab().toList().join(", ")
+                } else {""}
+            } else {
+                Show.show(scope.expectingTable) + " " + Show.show(scope.beforeDot)
+            }
+        )
+        scope.symbols.each {symbol =>
+            system.writeLine(Show.show(symbol))
+        }
+    }
+    let sql = "WITH c as (select x, y z from p, q.w) SELECT *, f.g(h) i FROM s.t1 a WHERE ((SELECT * FROM t2 b WHERE a.id = b.id))"
+    system.writeLine("")
+    system.writeLine("Tokenize:")
+    tokenize(sql).each {token =>
+        system.writeLine(showToken(token))
+    }
+    showCompletionScope(sql, "x")
+    showCompletionScope(sql, "*")
+    showCompletionScope(sql, "id")
+    showCompletionScope(sql, "t1 a")
+    showCompletionScope(sql, "1 a")
+    showCompletionScope(sql, " a WHERE")
+    showCompletionScope("insert into foo (a[1], b.x, c, d)", "c,")
+    showCompletionScope("insert into foo ()", ")")
+    showCompletionScope("update foo set --", "--")
+    showCompletionScope("update foo set x = 1, (y, z) = row (2, 3), w =", "w =")
+    showCompletionScope("update foo set answers = 1, --", "--")
+    showCompletionScope("merge into foo using (values (1)) as s(id) on id when matched then update set x =", "x =")
+}