amalgalite 1.0.0-x86-mingw32 → 1.1.1-x86-mingw32

data/HISTORY.rdoc

@@ -1,4 +1,20 @@
 = Amalgalite Changelog
+== Version 1.1.1 - 2011-03-27
+
+=== Enhancements
+
+* Update to SQLite 3.7.5
+* Enable SQLite Full Text Search compile time options FTS3 and FTS4 [github issue #10]
+  http://www.sqlite.org/fts3.html
+* Enable STAT2 SQLite compile time option to improve ANALYZE command support
+  http://www.sqlite.org/lang_analyze.html
+
+=== Bug Fixes
+
+* The C coded generated by gen_constants.rb was updated to support older
+  compilers [github issue #8] (patch from josb)
+* Fix 'amalgalite-pack --self' on ruby 1.9
+
 == Version 1.0.0 - 2011-01-16
 
 === Enhancements

data/bin/amalgalite-pack

@@ -1,7 +1,6 @@
 #!/usr/bin/env ruby
 require 'optparse'
 require 'pathname'
-require 'rubygems'
 
 #
 # add relative paths to the load path if we are not a gem and calculate what the
@@ -16,14 +15,36 @@ if  ".." == rel_path.to_s.split( File::SEPARATOR ).first then
   $:.unshift File.join( File.dirname( __FILE__ ), "../ext" )
 end
 
-# 
+#
 # snapshot of what is needed for amalgalite requires, this info may only be used
 # when packing amalgalite itself
 #
 loaded_features_before = $LOADED_FEATURES.dup
 require 'amalgalite/packer'
 loaded_features_after  = $LOADED_FEATURES.dup
-amalgalite_needs = loaded_features_after - loaded_features_before
+load_diff = loaded_features_after - loaded_features_before
+
+#
+# strip off any LOAD_PATH elements from the front of load_diff since that
+# will conflict with Amalgalite::Packer.amalgalite_require_order.  Also
+# strip out any 'rubygems' items since those are not used by Amalgalite
+# and show as a side effect fo the "require 'amalgalite/packer'"
+#
+strip_paths = $LOAD_PATH.sort.reverse
+amalgalite_needs = []
+load_diff.each do |f|
+  next if f.split( File::SEPARATOR ).include?( "rubygems" )
+  appended = false
+  strip_paths.each do |path|
+    if 0 == f.index(path ) then
+      rel_path = f.sub( path, '' ).sub(%r{\A#{File::SEPARATOR}},'')
+      amalgalite_needs <<  rel_path
+      appended = true
+      break
+    end
+  end
+  amalgalite_needs << f unless appended
+end
 
 #
 # Commandline parser
@@ -77,7 +98,7 @@ begin
   require 'amalgalite/packer'
   file_list = ARGV.dup
 
-  
+
   if options[:self] then
     options[:table_name] = Amalgalite::Requires::Bootstrap::DEFAULT_BOOTSTRAP_TABLE
     core_libs = (amalgalite_needs - Amalgalite::Packer.amalgalite_require_order).delete_if { |l| l.index(".rb").nil? }
@@ -85,9 +106,9 @@ begin
     #
     # check and make sure nothing is missed
     #
-    core_libs.each do |l| 
-      if l.index("malgalite") then
-        STDERR.puts "ERROR!  require_order needs an update #{l}" 
+    core_libs.each do |l|
+      if l.index("amalgalite") then
+        STDERR.puts "ERROR!  require_order needs an update #{l}"
         exit 2
       end
     end
@@ -116,7 +137,7 @@ Packing complete.  To utilize the bootstrapping in #{dbfile} you must do
 one of the following:
 
   * statically compile the amalgalite C extension into your application
-  * require 'amalgalite/amalgalite3'
+  * require 'amalgalite/#{RUBY_VERSION.sub(/\.\d$/,'')}/amalgalite3'
 
 Once one of those is working you can boostrap the Amalgalite library with
 this line in your code:

data/examples/fts3.rb

@@ -0,0 +1,144 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'amalgalite'
+require 'benchmark'
+
+begin
+  require 'json'
+rescue LoadError 
+  abort "'gem install json' to run this example"
+end
+
+
+README = <<_
+ This example programs assumes that you have downloaded the 'Tweets During the
+ State of the Union address' dataset from infochimps.com
+
+   http://www.infochimps.com/datasets/tweets-during-state-of-the-union-address
+
+ Please:
+   1) download this dataset
+   2) bunzip + untar this file
+   3) record the location of the 'twitter_stream.txt' file that is in the
+      tarball.  
+   4) Pass this file as the first parameter to this script.
+_
+
+twitter_stream = ARGV.shift
+abort README unless twitter_stream and File.readable?( twitter_stream )
+
+#
+# Lets create a database that utilizes FTS3 http://www.sqlite.org/fts3.html
+#
+#
+
+#
+# Create a database, this will create the DB if it doesn't exist
+#
+puts "Opening database (version #{Amalgalite::Version})"
+db = Amalgalite::Database.new("fts3.db")
+
+#
+# Create the schema unless it already exists in the table.  The meta information
+# about the database schema is available as the result of the db.schema method
+#
+schema = db.schema
+unless schema.tables['search'] 
+  puts "Create schema"
+  db.execute_batch <<-SQL
+  CREATE VIRTUAL TABLE search USING fts3(
+    filename VARCHAR(128),
+    content  TEXT 
+  );
+
+  CREATE TABLE plain (
+    filename VARCHAR(128),
+    content  TEXT 
+  );
+  SQL
+  db.reload_schema!
+end
+
+#
+# Only load the data if the db is empty
+#
+if db.first_value_from( "SELECT count(*) from search" ) == 0 then
+  before = Time.now
+  # 
+  # Load the tweets from the file passed on the commandline into the database
+  # We just want the text and the tweet id and insert that into the database.
+  #
+
+  lines = IO.readlines( twitter_stream )
+  idx = 0
+
+  # Inserting bulk rows as a transaction is good practice with SQLite, it is
+  # MUCH faster.  
+  db.transaction do |db_in_transaction|
+    lines.each do |line|
+
+      # quick little parse of the tweet
+      json = JSON.parse( line )
+      insert_data = {}
+      insert_data[':fname']   = json['id']
+      insert_data[':content'] = json['text']
+
+      # insert into the FTS3 table
+      db_in_transaction.prepare("INSERT INTO search( filename, content ) VALUES( :fname, :content );") do |stmt|
+        stmt.execute( insert_data )
+      end
+
+      # insert into the normal table for comparison
+      db_in_transaction.prepare("INSERT INTO plain( filename, content ) VALUES( :fname, :content );") do |stmt|
+        stmt.execute( insert_data )
+      end
+
+      idx += 1
+      print "Processed #{idx} of #{lines.size}\r"
+      $stdout.flush
+    end
+    puts "Finalizing..."
+  end
+  puts "Took #{Time.now - before} seconds to insert #{idx} lines of #{lines.size}"
+  puts "Done Inserting"
+end
+
+doc_count = db.first_value_from( "SELECT count(*) from search" ) 
+
+#
+# Now lets do some searching for some various words
+#
+
+%w[ president salmon thanks ].each do |word|
+
+  count = 100
+  puts
+  puts "Searching for '#{word}' across #{doc_count} tweets"
+  puts "=" * 60
+
+  Benchmark.bm( 15 ) do |x|
+
+    #
+    # search using the fts search to get the cont of tweets with the given word
+    #
+    x.report('fts3: ') do 
+      db.prepare( "SELECT count(filename) FROM search WHERE content MATCH '#{word}'" ) do |stmt|
+        count.times do
+          stmt.execute
+        end
+      end
+    end
+
+    #
+    # search using basic string matching in sqlite.  
+    #
+    x.report('plain: ') do 
+      db.prepare( "SELECT count(filename) FROM plain WHERE content LIKE '% #{word} %'" ) do |stmt|
+        count.times do
+          stmt.execute
+        end
+      end
+    end
+  end
+end
+

data/ext/amalgalite/amalgalite3_constants.c

@@ -17,10 +17,40 @@ void Init_amalgalite3_constants( )
      */
     VALUE mC = rb_define_module_under( mas, "Constants");
     /**
-     * module encapsulating the SQLite3 C extension constants for Config 
+     * module encapsulating the SQLite3 C extension constants for Config
      */
     VALUE mC_Config = rb_define_module_under(mC, "Config");
 
+    /**
+     * module encapsulating the SQLite3 C extension constants for DBStatus
+     */
+    VALUE mC_DBStatus = rb_define_module_under(mC, "DBStatus");
+
+    /**
+     * module encapsulating the SQLite3 C extension constants for DataType
+     */
+    VALUE mC_DataType = rb_define_module_under(mC, "DataType");
+
+    /**
+     * module encapsulating the SQLite3 C extension constants for Open
+     */
+    VALUE mC_Open = rb_define_module_under(mC, "Open");
+
+    /**
+     * module encapsulating the SQLite3 C extension constants for ResultCode
+     */
+    VALUE mC_ResultCode = rb_define_module_under(mC, "ResultCode");
+
+    /**
+     * module encapsulating the SQLite3 C extension constants for StatementStatus
+     */
+    VALUE mC_StatementStatus = rb_define_module_under(mC, "StatementStatus");
+
+    /**
+     * module encapsulating the SQLite3 C extension constants for Status
+     */
+    VALUE mC_Status = rb_define_module_under(mC, "Status");
+
     /* no meaningful autogenerated documentation -- constant is self explanatory ?*/
     rb_define_const(mC_Config, "GETMALLOC", INT2FIX(SQLITE_CONFIG_GETMALLOC));
 
@@ -66,11 +96,6 @@ void Init_amalgalite3_constants( )
     /* no meaningful autogenerated documentation -- constant is self explanatory ?*/
     rb_define_const(mC_Config, "SINGLETHREAD", INT2FIX(SQLITE_CONFIG_SINGLETHREAD));
 
-    /**
-     * module encapsulating the SQLite3 C extension constants for DBStatus 
-     */
-    VALUE mC_DBStatus = rb_define_module_under(mC, "DBStatus");
-
     /* no meaningful autogenerated documentation -- constant is self explanatory ?*/
     rb_define_const(mC_DBStatus, "CACHE_USED", INT2FIX(SQLITE_DBSTATUS_CACHE_USED));
 
@@ -86,11 +111,6 @@ void Init_amalgalite3_constants( )
     /* no meaningful autogenerated documentation -- constant is self explanatory ?*/
     rb_define_const(mC_DBStatus, "STMT_USED", INT2FIX(SQLITE_DBSTATUS_STMT_USED));
 
-    /**
-     * module encapsulating the SQLite3 C extension constants for DataType 
-     */
-    VALUE mC_DataType = rb_define_module_under(mC, "DataType");
-
     /* no meaningful autogenerated documentation -- constant is self explanatory ?*/
     rb_define_const(mC_DataType, "BLOB", INT2FIX(SQLITE_BLOB));
 
@@ -106,11 +126,6 @@ void Init_amalgalite3_constants( )
     /* no meaningful autogenerated documentation -- constant is self explanatory ?*/
     rb_define_const(mC_DataType, "TEXT", INT2FIX(SQLITE_TEXT));
 
-    /**
-     * module encapsulating the SQLite3 C extension constants for Open 
-     */
-    VALUE mC_Open = rb_define_module_under(mC, "Open");
-
     /* no meaningful autogenerated documentation -- constant is self explanatory ?*/
     rb_define_const(mC_Open, "AUTOPROXY", INT2FIX(SQLITE_OPEN_AUTOPROXY));
 
@@ -165,11 +180,6 @@ void Init_amalgalite3_constants( )
     /* no meaningful autogenerated documentation -- constant is self explanatory ?*/
     rb_define_const(mC_Open, "WAL", INT2FIX(SQLITE_OPEN_WAL));
 
-    /**
-     * module encapsulating the SQLite3 C extension constants for ResultCode 
-     */
-    VALUE mC_ResultCode = rb_define_module_under(mC, "ResultCode");
-
     /*  4 -- Callback routine requested an abort */
     rb_define_const(mC_ResultCode, "ABORT", INT2FIX(SQLITE_ABORT));
 
@@ -299,7 +309,7 @@ void Init_amalgalite3_constants( )
     /*  26 -- File opened that is not a database file */
     rb_define_const(mC_ResultCode, "NOTADB", INT2FIX(SQLITE_NOTADB));
 
-    /*  12 -- NOT USED. Table or record not found */
+    /*  12 -- Unknown opcode in sqlite3_file_control() */
     rb_define_const(mC_ResultCode, "NOTFOUND", INT2FIX(SQLITE_NOTFOUND));
 
     /* no meaningful autogenerated documentation -- constant is self explanatory ?*/
@@ -326,11 +336,6 @@ void Init_amalgalite3_constants( )
     /*  18 -- String or BLOB exceeds size limit */
     rb_define_const(mC_ResultCode, "TOOBIG", INT2FIX(SQLITE_TOOBIG));
 
-    /**
-     * module encapsulating the SQLite3 C extension constants for StatementStatus 
-     */
-    VALUE mC_StatementStatus = rb_define_module_under(mC, "StatementStatus");
-
     /* no meaningful autogenerated documentation -- constant is self explanatory ?*/
     rb_define_const(mC_StatementStatus, "AUTOINDEX", INT2FIX(SQLITE_STMTSTATUS_AUTOINDEX));
 
@@ -340,11 +345,6 @@ void Init_amalgalite3_constants( )
     /* no meaningful autogenerated documentation -- constant is self explanatory ?*/
     rb_define_const(mC_StatementStatus, "SORT", INT2FIX(SQLITE_STMTSTATUS_SORT));
 
-    /**
-     * module encapsulating the SQLite3 C extension constants for Status 
-     */
-    VALUE mC_Status = rb_define_module_under(mC, "Status");
-
     /* no meaningful autogenerated documentation -- constant is self explanatory ?*/
     rb_define_const(mC_Status, "MALLOC_COUNT", INT2FIX(SQLITE_STATUS_MALLOC_COUNT));
 

data/ext/amalgalite/extconf.rb

@@ -8,6 +8,9 @@ $ruby = ARGV.shift if ARGV[0]
 # make available table and column meta data api
 $CFLAGS += " -DSQLITE_ENABLE_COLUMN_METADATA=1"  
 $CFLAGS += " -DSQLITE_ENABLE_RTREE=1"
+$CFLAGS += " -DSQLITE_ENABLE_FTS3=1"
+$CFLAGS += " -DSQLITE_ENABLE_FTS3_PARENTHESIS=1"
+$CFLAGS += " -DSQLITE_ENABLE_STAT2=1"
 
 # we compile sqlite the same way that the installation of ruby is compiled.
 if RbConfig::MAKEFILE_CONFIG['configure_args'].include?( "--enable-pthread" ) then

data/ext/amalgalite/gen_constants.rb

@@ -170,10 +170,14 @@ File.open(fname, "w+") do |f|
   CONSTANTS.keys.sort.each do |klass|
     const_set = CONSTANTS[klass]
     f.puts "    /**"
-    f.puts "     * module encapsulating the SQLite3 C extension constants for #{klass} "
+    f.puts "     * module encapsulating the SQLite3 C extension constants for #{klass}"
     f.puts "     */"
     f.puts "    VALUE mC_#{klass} = rb_define_module_under(mC, \"#{klass}\");"
     f.puts 
+  end
+
+  CONSTANTS.keys.sort.each do |klass|
+    const_set = CONSTANTS[klass]
     const_set.keys.sort.each do |k|
       sql_const = const_set[k]
       const_doc = "    /* no meaningful autogenerated documentation -- constant is self explanatory ?*/" 

data/ext/amalgalite/sqlite3.c

@@ -1,6 +1,6 @@
 /******************************************************************************
 ** This file is an amalgamation of many separate C source files from SQLite
-** version 3.7.4.  By combining all the individual C code files into this 
+** version 3.7.5.  By combining all the individual C code files into this 
 ** single large file, the entire code can be compiled as a one translation
 ** unit.  This allows many compilers to do optimizations that would not be
 ** possible if the files were compiled separately.  Performance improvements
@@ -650,9 +650,9 @@ extern "C" {
 ** [sqlite3_libversion_number()], [sqlite3_sourceid()],
 ** [sqlite_version()] and [sqlite_source_id()].
 */
-#define SQLITE_VERSION        "3.7.4"
-#define SQLITE_VERSION_NUMBER 3007004
-#define SQLITE_SOURCE_ID      "2010-12-07 20:14:09 a586a4deeb25330037a49df295b36aaf624d0f45"
+#define SQLITE_VERSION        "3.7.5"
+#define SQLITE_VERSION_NUMBER 3007005
+#define SQLITE_SOURCE_ID      "2011-01-28 17:03:50 ed759d5a9edb3bba5f48f243df47be29e3fe8cd7"
 
 /*
 ** CAPI3REF: Run-Time Library Version Numbers
@@ -933,7 +933,7 @@ SQLITE_API int sqlite3_exec(
 #define SQLITE_INTERRUPT    9   /* Operation terminated by sqlite3_interrupt()*/
 #define SQLITE_IOERR       10   /* Some kind of disk I/O error occurred */
 #define SQLITE_CORRUPT     11   /* The database disk image is malformed */
-#define SQLITE_NOTFOUND    12   /* NOT USED. Table or record not found */
+#define SQLITE_NOTFOUND    12   /* Unknown opcode in sqlite3_file_control() */
 #define SQLITE_FULL        13   /* Insertion failed because database is full */
 #define SQLITE_CANTOPEN    14   /* Unable to open the database file */
 #define SQLITE_PROTOCOL    15   /* Database lock protocol error */
@@ -1165,7 +1165,9 @@ struct sqlite3_file {
 ** core reserves all opcodes less than 100 for its own use.
 ** A [SQLITE_FCNTL_LOCKSTATE | list of opcodes] less than 100 is available.
 ** Applications that define a custom xFileControl method should use opcodes
-** greater than 100 to avoid conflicts.
+** greater than 100 to avoid conflicts.  VFS implementations should
+** return [SQLITE_NOTFOUND] for file control opcodes that they do not
+** recognize.
 **
 ** The xSectorSize() method returns the sector size of the
 ** device that underlies the file.  The sector size is the
@@ -1258,6 +1260,21 @@ struct sqlite3_io_methods {
 ** for the nominated database. Allocating database file space in large
 ** chunks (say 1MB at a time), may reduce file-system fragmentation and
 ** improve performance on some systems.
+**
+** The [SQLITE_FCNTL_FILE_POINTER] opcode is used to obtain a pointer
+** to the [sqlite3_file] object associated with a particular database
+** connection.  See the [sqlite3_file_control()] documentation for
+** additional information.
+**
+** ^(The [SQLITE_FCNTL_SYNC_OMITTED] opcode is generated internally by
+** SQLite and sent to all VFSes in place of a call to the xSync method
+** when the database connection has [PRAGMA synchronous] set to OFF.)^
+** Some specialized VFSes need this signal in order to operate correctly
+** when [PRAGMA synchronous | PRAGMA synchronous=OFF] is set, but most 
+** VFSes do not need this signal and should silently ignore this opcode.
+** Applications should not call [sqlite3_file_control()] with this
+** opcode as doing so may disrupt the operation of the specilized VFSes
+** that do require it.  
 */
 #define SQLITE_FCNTL_LOCKSTATE        1
 #define SQLITE_GET_LOCKPROXYFILE      2
@@ -1266,6 +1283,7 @@ struct sqlite3_io_methods {
 #define SQLITE_FCNTL_SIZE_HINT        5
 #define SQLITE_FCNTL_CHUNK_SIZE       6
 #define SQLITE_FCNTL_FILE_POINTER     7
+#define SQLITE_FCNTL_SYNC_OMITTED     8
 
 
 /*
@@ -2385,7 +2403,7 @@ SQLITE_API void sqlite3_free_table(char **result);
 ** NULL pointer if [sqlite3_malloc()] is unable to allocate enough
 ** memory to hold the resulting string.
 **
-** ^(In sqlite3_snprintf() routine is similar to "snprintf()" from
+** ^(The sqlite3_snprintf() routine is similar to "snprintf()" from
 ** the standard C library.  The result is written into the
 ** buffer supplied as the second parameter whose size is given by
 ** the first parameter. Note that the order of the
@@ -2404,6 +2422,8 @@ SQLITE_API void sqlite3_free_table(char **result);
 ** the zero terminator.  So the longest string that can be completely
 ** written will be n-1 characters.
 **
+** ^The sqlite3_vsnprintf() routine is a varargs version of sqlite3_snprintf().
+**
 ** These routines all implement some additional formatting
 ** options that are useful for constructing SQL statements.
 ** All of the usual printf() formatting options apply.  In addition, there
@@ -2467,6 +2487,7 @@ SQLITE_API void sqlite3_free_table(char **result);
 SQLITE_API char *sqlite3_mprintf(const char*,...);
 SQLITE_API char *sqlite3_vmprintf(const char*, va_list);
 SQLITE_API char *sqlite3_snprintf(int,char*,const char*, ...);
+SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list);
 
 /*
 ** CAPI3REF: Memory Allocation Subsystem
@@ -2844,7 +2865,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
 ** case the database must already exist, otherwise an error is returned.</dd>)^
 **
 ** ^(<dt>[SQLITE_OPEN_READWRITE] | [SQLITE_OPEN_CREATE]</dt>
-** <dd>The database is opened for reading and writing, and is creates it if
+** <dd>The database is opened for reading and writing, and is created if
 ** it does not already exist. This is the behavior that is always used for
 ** sqlite3_open() and sqlite3_open16().</dd>)^
 ** </dl>
@@ -3193,14 +3214,31 @@ SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt);
 /*
 ** CAPI3REF: Determine If An SQL Statement Writes The Database
 **
-** ^The sqlite3_stmt_readonly(X) interface returns true (non-zero) if
-** the [prepared statement] X is [SELECT] statement and false (zero) if
-** X is an [INSERT], [UPDATE], [DELETE], CREATE, DROP, [ANALYZE],
-** [ALTER], or [REINDEX] statement.
-** If X is a NULL pointer or any other kind of statement, including but
-** not limited to [ATTACH], [DETACH], [COMMIT], [ROLLBACK], [RELEASE],
-** [SAVEPOINT], [PRAGMA], or [VACUUM] the result of sqlite3_stmt_readonly(X) is
-** undefined.
+** ^The sqlite3_stmt_readonly(X) interface returns true (non-zero) if 
+** and only if the [prepared statement] X makes no direct changes to
+** the content of the database file.
+**
+** Note that [application-defined SQL functions] or
+** [virtual tables] might change the database indirectly as a side effect.  
+** ^(For example, if an application defines a function "eval()" that 
+** calls [sqlite3_exec()], then the following SQL statement would
+** change the database file through side-effects:
+**
+** <blockquote><pre>
+**    SELECT eval('DELETE FROM t1') FROM t2;
+** </pre></blockquote>
+**
+** But because the [SELECT] statement does not change the database file
+** directly, sqlite3_stmt_readonly() would still return true.)^
+**
+** ^Transaction control statements such as [BEGIN], [COMMIT], [ROLLBACK],
+** [SAVEPOINT], and [RELEASE] cause sqlite3_stmt_readonly() to return true,
+** since the statements themselves do not actually modify the database but
+** rather they control the timing of when other statements modify the 
+** database.  ^The [ATTACH] and [DETACH] statements also cause
+** sqlite3_stmt_readonly() to return true since, while those statements
+** change the configuration of a database connection, they do not make 
+** changes to the content of the database files on disk.
 */
 SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt);
 
@@ -3594,13 +3632,17 @@ SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt*,int);
 ** be the case that the same database connection is being used by two or
 ** more threads at the same moment in time.
 **
-** For all versions of SQLite up to and including 3.6.23.1, it was required
-** after sqlite3_step() returned anything other than [SQLITE_ROW] that
-** [sqlite3_reset()] be called before any subsequent invocation of
-** sqlite3_step().  Failure to invoke [sqlite3_reset()] in this way would
-** result in an [SQLITE_MISUSE] return from sqlite3_step().  But after
-** version 3.6.23.1, sqlite3_step() began calling [sqlite3_reset()] 
-** automatically in this circumstance rather than returning [SQLITE_MISUSE].  
+** For all versions of SQLite up to and including 3.6.23.1, a call to
+** [sqlite3_reset()] was required after sqlite3_step() returned anything
+** other than [SQLITE_ROW] before any subsequent invocation of
+** sqlite3_step().  Failure to reset the prepared statement using 
+** [sqlite3_reset()] would result in an [SQLITE_MISUSE] return from
+** sqlite3_step().  But after version 3.6.23.1, sqlite3_step() began
+** calling [sqlite3_reset()] automatically in this circumstance rather
+** than returning [SQLITE_MISUSE].  This is not considered a compatibility
+** break because any application that ever receives an SQLITE_MISUSE error
+** is broken by definition.  The [SQLITE_OMIT_AUTORESET] compile-time option
+** can be used to restore the legacy behavior.
 **
 ** <b>Goofy Interface Alert:</b> In the legacy interface, the sqlite3_step()
 ** API always returns a generic error code, [SQLITE_ERROR], following any
@@ -3937,7 +3979,7 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt);
 ** ^(The fifth parameter is an arbitrary pointer.  The implementation of the
 ** function can gain access to this pointer using [sqlite3_user_data()].)^
 **
-** ^The seventh, eighth and ninth parameters, xFunc, xStep and xFinal, are
+** ^The sixth, seventh and eighth parameters, xFunc, xStep and xFinal, are
 ** pointers to C-language functions that implement the SQL function or
 ** aggregate. ^A scalar SQL function requires an implementation of the xFunc
 ** callback only; NULL pointers must be passed as the xStep and xFinal
@@ -3946,7 +3988,7 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt);
 ** SQL function or aggregate, pass NULL poiners for all three function
 ** callbacks.
 **
-** ^(If the tenth parameter to sqlite3_create_function_v2() is not NULL,
+** ^(If the ninth parameter to sqlite3_create_function_v2() is not NULL,
 ** then it is destructor for the application data pointer. 
 ** The destructor is invoked when the function is deleted, either by being
 ** overloaded or when the database connection closes.)^
@@ -4050,7 +4092,7 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6
 ** The xFunc (for scalar functions) or xStep (for aggregates) parameters
 ** to [sqlite3_create_function()] and [sqlite3_create_function16()]
 ** define callbacks that implement the SQL functions and aggregates.
-** The 4th parameter to these callbacks is an array of pointers to
+** The 3rd parameter to these callbacks is an array of pointers to
 ** [protected sqlite3_value] objects.  There is one [sqlite3_value] object for
 ** each parameter to the SQL function.  These routines are used to
 ** extract values from the [sqlite3_value] objects.
@@ -5778,7 +5820,8 @@ SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex*);
 #define SQLITE_MUTEX_STATIC_OPEN      4  /* sqlite3BtreeOpen() */
 #define SQLITE_MUTEX_STATIC_PRNG      5  /* sqlite3_random() */
 #define SQLITE_MUTEX_STATIC_LRU       6  /* lru page list */
-#define SQLITE_MUTEX_STATIC_LRU2      7  /* lru page list */
+#define SQLITE_MUTEX_STATIC_LRU2      7  /* NOT USED */
+#define SQLITE_MUTEX_STATIC_PMEM      7  /* sqlite3PageMalloc() */
 
 /*
 ** CAPI3REF: Retrieve the mutex for a database connection
@@ -5929,7 +5972,8 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF
 ** The value written into the *pCurrent parameter is undefined.</dd>)^
 **
 ** ^(<dt>SQLITE_STATUS_MALLOC_COUNT</dt>
-** <dd>This parameter records the number of separate memory allocations.</dd>)^
+** <dd>This parameter records the number of separate memory allocations
+** currently checked out.</dd>)^
 **
 ** ^(<dt>SQLITE_STATUS_PAGECACHE_USED</dt>
 ** <dd>This parameter returns the number of pages used out of the
@@ -6035,6 +6079,28 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r
 ** <dd>This parameter returns the number of lookaside memory slots currently
 ** checked out.</dd>)^
 **
+** ^(<dt>SQLITE_DBSTATUS_LOOKASIDE_HIT</dt>
+** <dd>This parameter returns the number malloc attempts that were 
+** satisfied using lookaside memory. Only the high-water value is meaningful;
+** the current value is always zero.
+** checked out.</dd>)^
+**
+** ^(<dt>SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE</dt>
+** <dd>This parameter returns the number malloc attempts that might have
+** been satisfied using lookaside memory but failed due to the amount of
+** memory requested being larger than the lookaside slot size.
+** Only the high-water value is meaningful;
+** the current value is always zero.
+** checked out.</dd>)^
+**
+** ^(<dt>SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL</dt>
+** <dd>This parameter returns the number malloc attempts that might have
+** been satisfied using lookaside memory but failed due to all lookaside
+** memory already being in use.
+** Only the high-water value is meaningful;
+** the current value is always zero.
+** checked out.</dd>)^
+**
 ** ^(<dt>SQLITE_DBSTATUS_CACHE_USED</dt>
 ** <dd>This parameter returns the approximate number of of bytes of heap
 ** memory used by all pager caches associated with the database connection.)^
@@ -6057,11 +6123,14 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r
 ** </dd>
 ** </dl>
 */
-#define SQLITE_DBSTATUS_LOOKASIDE_USED     0
-#define SQLITE_DBSTATUS_CACHE_USED         1
-#define SQLITE_DBSTATUS_SCHEMA_USED        2
-#define SQLITE_DBSTATUS_STMT_USED          3
-#define SQLITE_DBSTATUS_MAX                3   /* Largest defined DBSTATUS */
+#define SQLITE_DBSTATUS_LOOKASIDE_USED       0
+#define SQLITE_DBSTATUS_CACHE_USED           1
+#define SQLITE_DBSTATUS_SCHEMA_USED          2
+#define SQLITE_DBSTATUS_STMT_USED            3
+#define SQLITE_DBSTATUS_LOOKASIDE_HIT        4
+#define SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE  5
+#define SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL  6
+#define SQLITE_DBSTATUS_MAX                  6   /* Largest defined DBSTATUS */
 
 
 /*
@@ -6189,11 +6258,13 @@ typedef struct sqlite3_pcache sqlite3_pcache;
 ** first parameter, szPage, is the size in bytes of the pages that must
 ** be allocated by the cache.  ^szPage will not be a power of two.  ^szPage
 ** will the page size of the database file that is to be cached plus an
-** increment (here called "R") of about 100 or 200.  SQLite will use the
+** increment (here called "R") of less than 250.  SQLite will use the
 ** extra R bytes on each page to store metadata about the underlying
 ** database page on disk.  The value of R depends
 ** on the SQLite version, the target platform, and how SQLite was compiled.
-** ^R is constant for a particular build of SQLite.  ^The second argument to
+** ^(R is constant for a particular build of SQLite. Except, there are two
+** distinct values of R when SQLite is compiled with the proprietary
+** ZIPVFS extension.)^  ^The second argument to
 ** xCreate(), bPurgeable, is true if the cache being created will
 ** be used to cache database pages of a file stored on disk, or
 ** false if it is used for an in-memory database. The cache implementation
@@ -6225,7 +6296,7 @@ typedef struct sqlite3_pcache sqlite3_pcache;
 ** If the requested page is already in the page cache, then the page cache
 ** implementation must return a pointer to the page buffer with its content
 ** intact.  If the requested page is not already in the cache, then the
-** behavior of the cache implementation should use the value of the createFlag
+** cache implementation should use the value of the createFlag
 ** parameter to help it determined what action to take:
 **
 ** <table border=1 width=85% align=center>
@@ -6309,11 +6380,12 @@ typedef struct sqlite3_backup sqlite3_backup;
 **
 ** See Also: [Using the SQLite Online Backup API]
 **
-** ^Exclusive access is required to the destination database for the 
-** duration of the operation. ^However the source database is only
-** read-locked while it is actually being read; it is not locked
-** continuously for the entire backup operation. ^Thus, the backup may be
-** performed on a live source database without preventing other users from
+** ^SQLite holds a write transaction open on the destination database file
+** for the duration of the backup operation.
+** ^The source database is read-locked only while it is being read;
+** it is not locked continuously for the entire backup operation.
+** ^Thus, the backup may be performed on a live source database without
+** preventing other database connections from
 ** reading or writing to the source database while the backup is underway.
 ** 
 ** ^(To perform a backup operation: 
@@ -6340,11 +6412,11 @@ typedef struct sqlite3_backup sqlite3_backup;
 ** sqlite3_backup_init(D,N,S,M) identify the [database connection]
 ** and database name of the source database, respectively.
 ** ^The source and destination [database connections] (parameters S and D)
-** must be different or else sqlite3_backup_init(D,N,S,M) will file with
+** must be different or else sqlite3_backup_init(D,N,S,M) will fail with
 ** an error.
 **
 ** ^If an error occurs within sqlite3_backup_init(D,N,S,M), then NULL is
-** returned and an error code and error message are store3d in the
+** returned and an error code and error message are stored in the
 ** destination [database connection] D.
 ** ^The error code and message for the failed call to sqlite3_backup_init()
 ** can be retrieved using the [sqlite3_errcode()], [sqlite3_errmsg()], and/or
@@ -6361,7 +6433,7 @@ typedef struct sqlite3_backup sqlite3_backup;
 ** the source and destination databases specified by [sqlite3_backup] object B.
 ** ^If N is negative, all remaining source pages are copied. 
 ** ^If sqlite3_backup_step(B,N) successfully copies N pages and there
-** are still more pages to be copied, then the function resturns [SQLITE_OK].
+** are still more pages to be copied, then the function returns [SQLITE_OK].
 ** ^If sqlite3_backup_step(B,N) successfully finishes copying all pages
 ** from source to destination, then it returns [SQLITE_DONE].
 ** ^If an error occurs while running sqlite3_backup_step(B,N),
@@ -6375,7 +6447,7 @@ typedef struct sqlite3_backup sqlite3_backup;
 ** <li> the destination database was opened read-only, or
 ** <li> the destination database is using write-ahead-log journaling
 ** and the destination and source page sizes differ, or
-** <li> The destination database is an in-memory database and the
+** <li> the destination database is an in-memory database and the
 ** destination and source page sizes differ.
 ** </ol>)^
 **
@@ -6706,7 +6778,8 @@ SQLITE_API void *sqlite3_wal_hook(
 ** from SQL.
 **
 ** ^Every new [database connection] defaults to having the auto-checkpoint
-** enabled with a threshold of 1000 pages.  The use of this interface
+** enabled with a threshold of 1000 or [SQLITE_DEFAULT_WAL_AUTOCHECKPOINT]
+** pages.  The use of this interface
 ** is only necessary if the default setting is found to be suboptimal
 ** for a particular application.
 */
@@ -8877,6 +8950,7 @@ struct Lookaside {
   u8 bMalloced;           /* True if pStart obtained from sqlite3_malloc() */
   int nOut;               /* Number of buffers currently checked out */
   int mxOut;              /* Highwater mark for nOut */
+  int anStat[3];          /* 0: hits.  1: size misses.  2: full misses */
   LookasideSlot *pFree;   /* List of available buffers */
   void *pStart;           /* First byte of available memory space */
   void *pEnd;             /* First byte past end of available space */
@@ -8955,6 +9029,7 @@ struct sqlite3 {
   struct Vdbe *pVdbe;           /* List of active virtual machines */
   int activeVdbeCnt;            /* Number of VDBEs currently executing */
   int writeVdbeCnt;             /* Number of active VDBEs that are writing */
+  int vdbeExecCnt;              /* Number of nested calls to VdbeExec() */
   void (*xTrace)(void*,const char*);        /* Trace function */
   void *pTraceArg;                          /* Argument to the trace function */
   void (*xProfile)(void*,const char*,u64);  /* Profiling function */
@@ -11727,6 +11802,9 @@ static const char * const azCompileOpt[] = {
 #ifdef SQLITE_OMIT_AUTOMATIC_INDEX
   "OMIT_AUTOMATIC_INDEX",
 #endif
+#ifdef SQLITE_OMIT_AUTORESET
+  "OMIT_AUTORESET",
+#endif
 #ifdef SQLITE_OMIT_AUTOVACUUM
   "OMIT_AUTOVACUUM",
 #endif
@@ -12002,16 +12080,14 @@ typedef unsigned char Bool;
 ** 
 ** Every cursor that the virtual machine has open is represented by an
 ** instance of the following structure.
-**
-** If the VdbeCursor.isTriggerRow flag is set it means that this cursor is
-** really a single row that represents the NEW or OLD pseudo-table of
-** a row trigger.  The data for the row is stored in VdbeCursor.pData and
-** the rowid is in VdbeCursor.iKey.
 */
 struct VdbeCursor {
   BtCursor *pCursor;    /* The cursor structure of the backend */
+  Btree *pBt;           /* Separate file holding temporary table */
+  KeyInfo *pKeyInfo;    /* Info about index keys needed by index cursors */
   int iDb;              /* Index of cursor database in db->aDb[] (or -1) */
-  i64 lastRowid;        /* Last rowid from a Next or NextIdx operation */
+  int pseudoTableReg;   /* Register holding pseudotable content. */
+  int nField;           /* Number of fields in the header */
   Bool zeroed;          /* True if zeroed out and ready for reuse */
   Bool rowidIsValid;    /* True if lastRowid is valid */
   Bool atFirst;         /* True if pointing to first entry */
@@ -12021,14 +12097,11 @@ struct VdbeCursor {
   Bool isTable;         /* True if a table requiring integer keys */
   Bool isIndex;         /* True if an index containing keys only - no data */
   Bool isOrdered;       /* True if the underlying table is BTREE_UNORDERED */
-  i64 movetoTarget;     /* Argument to the deferred sqlite3BtreeMoveto() */
-  Btree *pBt;           /* Separate file holding temporary table */
-  int pseudoTableReg;   /* Register holding pseudotable content. */
-  KeyInfo *pKeyInfo;    /* Info about index keys needed by index cursors */
-  int nField;           /* Number of fields in the header */
-  i64 seqCount;         /* Sequence counter */
   sqlite3_vtab_cursor *pVtabCursor;  /* The cursor for a virtual table */
   const sqlite3_module *pModule;     /* Module for cursor pVtabCursor */
+  i64 seqCount;         /* Sequence counter */
+  i64 movetoTarget;     /* Argument to the deferred sqlite3BtreeMoveto() */
+  i64 lastRowid;        /* Last rowid from a Next or NextIdx operation */
 
   /* Result of last sqlite3BtreeMoveto() done by an OP_NotExists or 
   ** OP_IsUnique opcode on this cursor. */
@@ -12100,25 +12173,19 @@ struct VdbeFrame {
 /*
 ** Internally, the vdbe manipulates nearly all SQL values as Mem
 ** structures. Each Mem struct may cache multiple representations (string,
-** integer etc.) of the same value.  A value (and therefore Mem structure)
-** has the following properties:
-**
-** Each value has a manifest type. The manifest type of the value stored
-** in a Mem struct is returned by the MemType(Mem*) macro. The type is
-** one of SQLITE_NULL, SQLITE_INTEGER, SQLITE_REAL, SQLITE_TEXT or
-** SQLITE_BLOB.
+** integer etc.) of the same value.
 */
 struct Mem {
+  sqlite3 *db;        /* The associated database connection */
+  char *z;            /* String or BLOB value */
+  double r;           /* Real value */
   union {
-    i64 i;              /* Integer value. */
+    i64 i;              /* Integer value used when MEM_Int is set in flags */
     int nZero;          /* Used when bit MEM_Zero is set in flags */
     FuncDef *pDef;      /* Used only when flags==MEM_Agg */
     RowSet *pRowSet;    /* Used only when flags==MEM_RowSet */
     VdbeFrame *pFrame;  /* Used when flags==MEM_Frame */
   } u;
-  double r;           /* Real value */
-  sqlite3 *db;        /* The associated database connection */
-  char *z;            /* String or BLOB value */
   int n;              /* Number of characters in string value, excluding '\0' */
   u16 flags;          /* Some combination of MEM_Null, MEM_Str, MEM_Dyn, etc. */
   u8  type;           /* One of SQLITE_NULL, SQLITE_TEXT, SQLITE_INTEGER, etc */
@@ -12142,9 +12209,6 @@ struct Mem {
 ** database (see below for exceptions). If the MEM_Term flag is also
 ** set, then the string is nul terminated. The MEM_Int and MEM_Real 
 ** flags may coexist with the MEM_Str flag.
-**
-** Multiple of these values can appear in Mem.flags.  But only one
-** at a time can appear in Mem.type.
 */
 #define MEM_Null      0x0001   /* Value is NULL */
 #define MEM_Str       0x0002   /* Value is a string */
@@ -12227,23 +12291,11 @@ struct sqlite3_context {
   CollSeq *pColl;       /* Collating sequence */
 };
 
-/*
-** A Set structure is used for quick testing to see if a value
-** is part of a small set.  Sets are used to implement code like
-** this:
-**            x.y IN ('hi','hoo','hum')
-*/
-typedef struct Set Set;
-struct Set {
-  Hash hash;             /* A set is just a hash table */
-  HashElem *prev;        /* Previously accessed hash elemen */
-};
-
 /*
 ** An instance of the virtual machine.  This structure contains the complete
 ** state of the virtual machine.
 **
-** The "sqlite3_stmt" structure pointer that is returned by sqlite3_compile()
+** The "sqlite3_stmt" structure pointer that is returned by sqlite3_prepare()
 ** is really a pointer to an instance of this structure.
 **
 ** The Vdbe.inVtabMethod variable is set to non-zero for the duration of
@@ -12256,31 +12308,31 @@ struct Set {
 */
 struct Vdbe {
   sqlite3 *db;            /* The database connection that owns this statement */
-  Vdbe *pPrev,*pNext;     /* Linked list of VDBEs with the same Vdbe.db */
+  Op *aOp;                /* Space to hold the virtual machine's program */
+  Mem *aMem;              /* The memory locations */
+  Mem **apArg;            /* Arguments to currently executing user function */
+  Mem *aColName;          /* Column names to return */
+  Mem *pResultSet;        /* Pointer to an array of results */
+  int nMem;               /* Number of memory locations currently allocated */
   int nOp;                /* Number of instructions in the program */
   int nOpAlloc;           /* Number of slots allocated for aOp[] */
-  Op *aOp;                /* Space to hold the virtual machine's program */
   int nLabel;             /* Number of labels used */
   int nLabelAlloc;        /* Number of slots allocated in aLabel[] */
   int *aLabel;            /* Space to hold the labels */
-  Mem **apArg;            /* Arguments to currently executing user function */
-  Mem *aColName;          /* Column names to return */
-  Mem *pResultSet;        /* Pointer to an array of results */
   u16 nResColumn;         /* Number of columns in one row of the result set */
   u16 nCursor;            /* Number of slots in apCsr[] */
+  u32 magic;              /* Magic number for sanity checking */
+  char *zErrMsg;          /* Error message written here */
+  Vdbe *pPrev,*pNext;     /* Linked list of VDBEs with the same Vdbe.db */
   VdbeCursor **apCsr;     /* One element of this array for each open cursor */
-  u8 errorAction;         /* Recovery action to do in case of an error */
-  u8 okVar;               /* True if azVar[] has been initialized */
-  ynVar nVar;             /* Number of entries in aVar[] */
   Mem *aVar;              /* Values for the OP_Variable opcode. */
   char **azVar;           /* Name of variables */
-  u32 magic;              /* Magic number for sanity checking */
-  int nMem;               /* Number of memory locations currently allocated */
-  Mem *aMem;              /* The memory locations */
+  ynVar nVar;             /* Number of entries in aVar[] */
   u32 cacheCtr;           /* VdbeCursor row cache generation counter */
   int pc;                 /* The program counter */
   int rc;                 /* Value to return */
-  char *zErrMsg;          /* Error message written here */
+  u8 errorAction;         /* Recovery action to do in case of an error */
+  u8 okVar;               /* True if azVar[] has been initialized */
   u8 explain;             /* True if EXPLAIN present on SQL command */
   u8 changeCntOn;         /* True to update the change-counter */
   u8 expired;             /* True if the VM needs to be recompiled */
@@ -12292,14 +12344,16 @@ struct Vdbe {
   u8 isPrepareV2;         /* True if prepared with prepare_v2() */
   int nChange;            /* Number of db changes made since last reset */
   int btreeMask;          /* Bitmask of db->aDb[] entries referenced */
-  i64 startTime;          /* Time when query started - used for profiling */
-  BtreeMutexArray aMutex; /* An array of Btree used here and needing locks */
+  int iStatement;         /* Statement number (or 0 if has not opened stmt) */
   int aCounter[3];        /* Counters used by sqlite3_stmt_status() */
-  char *zSql;             /* Text of the SQL statement that generated this */
-  void *pFree;            /* Free this when deleting the vdbe */
+  BtreeMutexArray aMutex; /* An array of Btree used here and needing locks */
+#ifndef SQLITE_OMIT_TRACE
+  i64 startTime;          /* Time when query started - used for profiling */
+#endif
   i64 nFkConstraint;      /* Number of imm. FK constraints this VM */
   i64 nStmtDefCons;       /* Number of def. constraints when stmt started */
-  int iStatement;         /* Statement number (or 0 if has not opened stmt) */
+  char *zSql;             /* Text of the SQL statement that generated this */
+  void *pFree;            /* Free this when deleting the vdbe */
 #ifdef SQLITE_DEBUG
   FILE *trace;            /* Write an execution trace here, if not NULL */
 #endif
@@ -12509,6 +12563,22 @@ SQLITE_API int sqlite3_db_status(
       break;
     }
 
+    case SQLITE_DBSTATUS_LOOKASIDE_HIT:
+    case SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE:
+    case SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL: {
+      testcase( op==SQLITE_DBSTATUS_LOOKASIDE_HIT );
+      testcase( op==SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE );
+      testcase( op==SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL );
+      assert( (op-SQLITE_DBSTATUS_LOOKASIDE_HIT)>=0 );
+      assert( (op-SQLITE_DBSTATUS_LOOKASIDE_HIT)<3 );
+      *pCurrent = 0;
+      *pHighwater = db->lookaside.anStat[op - SQLITE_DBSTATUS_LOOKASIDE_HIT];
+      if( resetFlag ){
+        db->lookaside.anStat[op - SQLITE_DBSTATUS_LOOKASIDE_HIT] = 0;
+      }
+      break;
+    }
+
     /* 
     ** Return an approximation for the amount of memory currently used
     ** by all pagers associated with the given database connection.  The
@@ -16867,7 +16937,7 @@ static int pthreadMutexEnd(void){ return SQLITE_OK; }
 ** <li>  SQLITE_MUTEX_STATIC_MEM2
 ** <li>  SQLITE_MUTEX_STATIC_PRNG
 ** <li>  SQLITE_MUTEX_STATIC_LRU
-** <li>  SQLITE_MUTEX_STATIC_LRU2
+** <li>  SQLITE_MUTEX_STATIC_PMEM
 ** </ul>
 **
 ** The first two constants cause sqlite3_mutex_alloc() to create
@@ -17277,7 +17347,7 @@ static int winMutexEnd(void){
 ** <li>  SQLITE_MUTEX_STATIC_MEM2
 ** <li>  SQLITE_MUTEX_STATIC_PRNG
 ** <li>  SQLITE_MUTEX_STATIC_LRU
-** <li>  SQLITE_MUTEX_STATIC_LRU2
+** <li>  SQLITE_MUTEX_STATIC_PMEM
 ** </ul>
 **
 ** The first two constants cause sqlite3_mutex_alloc() to create
@@ -18070,14 +18140,20 @@ SQLITE_PRIVATE void *sqlite3DbMallocRaw(sqlite3 *db, int n){
     if( db->mallocFailed ){
       return 0;
     }
-    if( db->lookaside.bEnabled && n<=db->lookaside.sz
-         && (pBuf = db->lookaside.pFree)!=0 ){
-      db->lookaside.pFree = pBuf->pNext;
-      db->lookaside.nOut++;
-      if( db->lookaside.nOut>db->lookaside.mxOut ){
-        db->lookaside.mxOut = db->lookaside.nOut;
+    if( db->lookaside.bEnabled ){
+      if( n>db->lookaside.sz ){
+        db->lookaside.anStat[1]++;
+      }else if( (pBuf = db->lookaside.pFree)==0 ){
+        db->lookaside.anStat[2]++;
+      }else{
+        db->lookaside.pFree = pBuf->pNext;
+        db->lookaside.nOut++;
+        db->lookaside.anStat[0]++;
+        if( db->lookaside.nOut>db->lookaside.mxOut ){
+          db->lookaside.mxOut = db->lookaside.nOut;
+        }
+        return (void*)pBuf;
       }
-      return (void*)pBuf;
     }
   }
 #else
@@ -18989,6 +19065,7 @@ SQLITE_PRIVATE void sqlite3StrAccumAppend(StrAccum *p, const char *z, int N){
         return;
       }
     }else{
+      char *zOld = (p->zText==p->zBase ? 0 : p->zText);
       i64 szNew = p->nChar;
       szNew += N + 1;
       if( szNew > p->mxAlloc ){
@@ -18999,13 +19076,12 @@ SQLITE_PRIVATE void sqlite3StrAccumAppend(StrAccum *p, const char *z, int N){
         p->nAlloc = (int)szNew;
       }
       if( p->useMalloc==1 ){
-        zNew = sqlite3DbMallocRaw(p->db, p->nAlloc );
+        zNew = sqlite3DbRealloc(p->db, zOld, p->nAlloc);
       }else{
-        zNew = sqlite3_malloc(p->nAlloc);
+        zNew = sqlite3_realloc(zOld, p->nAlloc);
       }
       if( zNew ){
-        memcpy(zNew, p->zText, p->nChar);
-        sqlite3StrAccumReset(p);
+        if( zOld==0 ) memcpy(zNew, p->zText, p->nChar);
         p->zText = zNew;
       }else{
         p->mallocFailed = 1;
@@ -19160,21 +19236,28 @@ SQLITE_API char *sqlite3_mprintf(const char *zFormat, ...){
 ** current locale settings.  This is important for SQLite because we
 ** are not able to use a "," as the decimal point in place of "." as
 ** specified by some locales.
+**
+** Oops:  The first two arguments of sqlite3_snprintf() are backwards
+** from the snprintf() standard.  Unfortunately, it is too late to change
+** this without breaking compatibility, so we just have to live with the
+** mistake.
+**
+** sqlite3_vsnprintf() is the varargs version.
 */
-SQLITE_API char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){
-  char *z;
-  va_list ap;
+SQLITE_API char *sqlite3_vsnprintf(int n, char *zBuf, const char *zFormat, va_list ap){
   StrAccum acc;
-
-  if( n<=0 ){
-    return zBuf;
-  }
+  if( n<=0 ) return zBuf;
   sqlite3StrAccumInit(&acc, zBuf, n, 0);
   acc.useMalloc = 0;
-  va_start(ap,zFormat);
   sqlite3VXPrintf(&acc, 0, zFormat, ap);
+  return sqlite3StrAccumFinish(&acc);
+}
+SQLITE_API char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){
+  char *z;
+  va_list ap;
+  va_start(ap,zFormat);
+  z = sqlite3_vsnprintf(n, zBuf, zFormat, ap);
   va_end(ap);
-  z = sqlite3StrAccumFinish(&acc);
   return z;
 }
 
@@ -22189,7 +22272,7 @@ static int os2FileControl(sqlite3_file *id, int op, void *pArg){
       return SQLITE_OK;
     }
   }
-  return SQLITE_ERROR;
+  return SQLITE_NOTFOUND;
 }
 
 /*
@@ -22905,7 +22988,9 @@ SQLITE_API int sqlite3_os_end(void){
 #include <unistd.h>
 #include <sys/time.h>
 #include <errno.h>
+#ifndef SQLITE_OMIT_WAL
 #include <sys/mman.h>
+#endif
 
 #if SQLITE_ENABLE_LOCKING_STYLE
 # include <sys/ioctl.h>
@@ -26123,8 +26208,11 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){
       return proxyFileControl(id,op,pArg);
     }
 #endif /* SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) */
+    case SQLITE_FCNTL_SYNC_OMITTED: {
+      return SQLITE_OK;  /* A no-op */
+    }
   }
-  return SQLITE_ERROR;
+  return SQLITE_NOTFOUND;
 }
 
 /*
@@ -28393,18 +28481,19 @@ extern int gethostuuid(uuid_t id, const struct timespec *wait);
 ** bytes of writable memory.
 */
 static int proxyGetHostID(unsigned char *pHostID, int *pError){
-  struct timespec timeout = {1, 0}; /* 1 sec timeout */
-  
   assert(PROXY_HOSTIDLEN == sizeof(uuid_t));
   memset(pHostID, 0, PROXY_HOSTIDLEN);
 #if defined(__MAX_OS_X_VERSION_MIN_REQUIRED)\
                && __MAC_OS_X_VERSION_MIN_REQUIRED<1050
-  if( gethostuuid(pHostID, &timeout) ){
-    int err = errno;
-    if( pError ){
-      *pError = err;
+  {
+    static const struct timespec timeout = {1, 0}; /* 1 sec timeout */
+    if( gethostuuid(pHostID, &timeout) ){
+      int err = errno;
+      if( pError ){
+        *pError = err;
+      }
+      return SQLITE_IOERR;
     }
-    return SQLITE_IOERR;
   }
 #endif
 #ifdef SQLITE_TEST
@@ -30710,8 +30799,11 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){
       SimulateIOErrorBenign(0);
       return SQLITE_OK;
     }
+    case SQLITE_FCNTL_SYNC_OMITTED: {
+      return SQLITE_OK;
+    }
   }
-  return SQLITE_ERROR;
+  return SQLITE_NOTFOUND;
 }
 
 /*
@@ -33331,6 +33423,38 @@ SQLITE_PRIVATE void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHd
 typedef struct PCache1 PCache1;
 typedef struct PgHdr1 PgHdr1;
 typedef struct PgFreeslot PgFreeslot;
+typedef struct PGroup PGroup;
+
+/* Each page cache (or PCache) belongs to a PGroup.  A PGroup is a set 
+** of one or more PCaches that are able to recycle each others unpinned
+** pages when they are under memory pressure.  A PGroup is an instance of
+** the following object.
+**
+** This page cache implementation works in one of two modes:
+**
+**   (1)  Every PCache is the sole member of its own PGroup.  There is
+**        one PGroup per PCache.
+**
+**   (2)  There is a single global PGroup that all PCaches are a member
+**        of.
+**
+** Mode 1 uses more memory (since PCache instances are not able to rob
+** unused pages from other PCaches) but it also operates without a mutex,
+** and is therefore often faster.  Mode 2 requires a mutex in order to be
+** threadsafe, but is able recycle pages more efficient.
+**
+** For mode (1), PGroup.mutex is NULL.  For mode (2) there is only a single
+** PGroup which is the pcache1.grp global variable and its mutex is
+** SQLITE_MUTEX_STATIC_LRU.
+*/
+struct PGroup {
+  sqlite3_mutex *mutex;          /* MUTEX_STATIC_LRU or NULL */
+  int nMaxPage;                  /* Sum of nMax for purgeable caches */
+  int nMinPage;                  /* Sum of nMin for purgeable caches */
+  int mxPinned;                  /* nMaxpage + 10 - nMinPage */
+  int nCurrentPage;              /* Number of purgeable pages allocated */
+  PgHdr1 *pLruHead, *pLruTail;   /* LRU list of unpinned pages */
+};
 
 /* Each page cache is an instance of the following object.  Every
 ** open database file (including each in-memory database and each
@@ -33344,16 +33468,17 @@ struct PCache1 {
   /* Cache configuration parameters. Page size (szPage) and the purgeable
   ** flag (bPurgeable) are set when the cache is created. nMax may be 
   ** modified at any time by a call to the pcache1CacheSize() method.
-  ** The global mutex must be held when accessing nMax.
+  ** The PGroup mutex must be held when accessing nMax.
   */
+  PGroup *pGroup;                     /* PGroup this cache belongs to */
   int szPage;                         /* Size of allocated pages in bytes */
   int bPurgeable;                     /* True if cache is purgeable */
   unsigned int nMin;                  /* Minimum number of pages reserved */
   unsigned int nMax;                  /* Configured "cache_size" value */
+  unsigned int n90pct;                /* nMax*9/10 */
 
   /* Hash table of all pages. The following variables may only be accessed
-  ** when the accessor is holding the global mutex (see pcache1EnterMutex() 
-  ** and pcache1LeaveMutex()).
+  ** when the accessor is holding the PGroup mutex.
   */
   unsigned int nRecyclable;           /* Number of pages in the LRU list */
   unsigned int nPage;                 /* Total number of pages in apHash */
@@ -33389,21 +33514,27 @@ struct PgFreeslot {
 ** Global data used by this cache.
 */
 static SQLITE_WSD struct PCacheGlobal {
-  sqlite3_mutex *mutex;               /* static mutex MUTEX_STATIC_LRU */
-
-  int nMaxPage;                       /* Sum of nMaxPage for purgeable caches */
-  int nMinPage;                       /* Sum of nMinPage for purgeable caches */
-  int nCurrentPage;                   /* Number of purgeable pages allocated */
-  PgHdr1 *pLruHead, *pLruTail;        /* LRU list of unpinned pages */
-
-  /* Variables related to SQLITE_CONFIG_PAGECACHE settings. */
-  int szSlot;                         /* Size of each free slot */
-  int nSlot;                          /* The number of pcache slots */
-  int nFreeSlot;                      /* Number of unused pcache slots */
-  int nReserve;                       /* Try to keep nFreeSlot above this */
-  void *pStart, *pEnd;                /* Bounds of pagecache malloc range */
-  PgFreeslot *pFree;                  /* Free page blocks */
-  int isInit;                         /* True if initialized */
+  PGroup grp;                    /* The global PGroup for mode (2) */
+
+  /* Variables related to SQLITE_CONFIG_PAGECACHE settings.  The
+  ** szSlot, nSlot, pStart, pEnd, nReserve, and isInit values are all
+  ** fixed at sqlite3_initialize() time and do not require mutex protection.
+  ** The nFreeSlot and pFree values do require mutex protection.
+  */
+  int isInit;                    /* True if initialized */
+  int szSlot;                    /* Size of each free slot */
+  int nSlot;                     /* The number of pcache slots */
+  int nReserve;                  /* Try to keep nFreeSlot above this */
+  void *pStart, *pEnd;           /* Bounds of pagecache malloc range */
+  /* Above requires no mutex.  Use mutex below for variable that follow. */
+  sqlite3_mutex *mutex;          /* Mutex for accessing the following: */
+  int nFreeSlot;                 /* Number of unused pcache slots */
+  PgFreeslot *pFree;             /* Free page blocks */
+  /* The following value requires a mutex to change.  We skip the mutex on
+  ** reading because (1) most platforms read a 32-bit integer atomically and
+  ** (2) even if an incorrect value is read, no great harm is done since this
+  ** is really just an optimization. */
+  int bUnderPressure;            /* True if low on PAGECACHE memory */
 } pcache1_g;
 
 /*
@@ -33429,10 +33560,10 @@ static SQLITE_WSD struct PCacheGlobal {
 #define PAGE_TO_PGHDR1(c, p) (PgHdr1*)(((char*)p) + c->szPage)
 
 /*
-** Macros to enter and leave the global LRU mutex.
+** Macros to enter and leave the PCache LRU mutex.
 */
-#define pcache1EnterMutex() sqlite3_mutex_enter(pcache1.mutex)
-#define pcache1LeaveMutex() sqlite3_mutex_leave(pcache1.mutex)
+#define pcache1EnterMutex(X) sqlite3_mutex_enter((X)->mutex)
+#define pcache1LeaveMutex(X) sqlite3_mutex_leave((X)->mutex)
 
 /******************************************************************************/
 /******** Page Allocation/SQLITE_CONFIG_PCACHE Related Functions **************/
@@ -33442,6 +33573,9 @@ static SQLITE_WSD struct PCacheGlobal {
 ** supplied to use for the page-cache by passing the SQLITE_CONFIG_PAGECACHE
 ** verb to sqlite3_config(). Parameter pBuf points to an allocation large
 ** enough to contain 'n' buffers of 'sz' bytes each.
+**
+** This routine is called from sqlite3_initialize() and so it is guaranteed
+** to be serialized already.  There is no need for further mutexing.
 */
 SQLITE_PRIVATE void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){
   if( pcache1.isInit ){
@@ -33452,6 +33586,7 @@ SQLITE_PRIVATE void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){
     pcache1.nReserve = n>90 ? 10 : (n/10 + 1);
     pcache1.pStart = pBuf;
     pcache1.pFree = 0;
+    pcache1.bUnderPressure = 0;
     while( n-- ){
       p = (PgFreeslot*)pBuf;
       p->pNext = pcache1.pFree;
@@ -33467,32 +33602,36 @@ SQLITE_PRIVATE void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){
 ** configured using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no 
 ** such buffer exists or there is no space left in it, this function falls 
 ** back to sqlite3Malloc().
+**
+** Multiple threads can run this routine at the same time.  Global variables
+** in pcache1 need to be protected via mutex.
 */
 static void *pcache1Alloc(int nByte){
-  void *p;
-  assert( sqlite3_mutex_held(pcache1.mutex) );
+  void *p = 0;
+  assert( sqlite3_mutex_notheld(pcache1.grp.mutex) );
   sqlite3StatusSet(SQLITE_STATUS_PAGECACHE_SIZE, nByte);
-  if( nByte<=pcache1.szSlot && pcache1.pFree ){
-    assert( pcache1.isInit );
+  if( nByte<=pcache1.szSlot ){
+    sqlite3_mutex_enter(pcache1.mutex);
     p = (PgHdr1 *)pcache1.pFree;
-    pcache1.pFree = pcache1.pFree->pNext;
-    pcache1.nFreeSlot--;
-    assert( pcache1.nFreeSlot>=0 );
-    sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_USED, 1);
-  }else{
-
-    /* Allocate a new buffer using sqlite3Malloc. Before doing so, exit the
-    ** global pcache mutex and unlock the pager-cache object pCache. This is 
-    ** so that if the attempt to allocate a new buffer causes the the 
-    ** configured soft-heap-limit to be breached, it will be possible to
-    ** reclaim memory from this pager-cache.
+    if( p ){
+      pcache1.pFree = pcache1.pFree->pNext;
+      pcache1.nFreeSlot--;
+      pcache1.bUnderPressure = pcache1.nFreeSlot<pcache1.nReserve;
+      assert( pcache1.nFreeSlot>=0 );
+      sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_USED, 1);
+    }
+    sqlite3_mutex_leave(pcache1.mutex);
+  }
+  if( p==0 ){
+    /* Memory is not available in the SQLITE_CONFIG_PAGECACHE pool.  Get
+    ** it from sqlite3Malloc instead.
     */
-    pcache1LeaveMutex();
     p = sqlite3Malloc(nByte);
-    pcache1EnterMutex();
     if( p ){
       int sz = sqlite3MallocSize(p);
+      sqlite3_mutex_enter(pcache1.mutex);
       sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_OVERFLOW, sz);
+      sqlite3_mutex_leave(pcache1.mutex);
     }
     sqlite3MemdebugSetType(p, MEMTYPE_PCACHE);
   }
@@ -33503,22 +33642,26 @@ static void *pcache1Alloc(int nByte){
 ** Free an allocated buffer obtained from pcache1Alloc().
 */
 static void pcache1Free(void *p){
-  assert( sqlite3_mutex_held(pcache1.mutex) );
   if( p==0 ) return;
   if( p>=pcache1.pStart && p<pcache1.pEnd ){
     PgFreeslot *pSlot;
+    sqlite3_mutex_enter(pcache1.mutex);
     sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_USED, -1);
     pSlot = (PgFreeslot*)p;
     pSlot->pNext = pcache1.pFree;
     pcache1.pFree = pSlot;
     pcache1.nFreeSlot++;
+    pcache1.bUnderPressure = pcache1.nFreeSlot<pcache1.nReserve;
     assert( pcache1.nFreeSlot<=pcache1.nSlot );
+    sqlite3_mutex_leave(pcache1.mutex);
   }else{
     int iSize;
     assert( sqlite3MemdebugHasType(p, MEMTYPE_PCACHE) );
     sqlite3MemdebugSetType(p, MEMTYPE_HEAP);
     iSize = sqlite3MallocSize(p);
+    sqlite3_mutex_enter(pcache1.mutex);
     sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_OVERFLOW, -iSize);
+    sqlite3_mutex_leave(pcache1.mutex);
     sqlite3_free(p);
   }
 }
@@ -33528,7 +33671,6 @@ static void pcache1Free(void *p){
 ** Return the size of a pcache allocation
 */
 static int pcache1MemSize(void *p){
-  assert( sqlite3_mutex_held(pcache1.mutex) );
   if( p>=pcache1.pStart && p<pcache1.pEnd ){
     return pcache1.szSlot;
   }else{
@@ -33552,7 +33694,7 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache){
   if( pPg ){
     p = PAGE_TO_PGHDR1(pCache, pPg);
     if( pCache->bPurgeable ){
-      pcache1.nCurrentPage++;
+      pCache->pGroup->nCurrentPage++;
     }
   }else{
     p = 0;
@@ -33569,8 +33711,9 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache){
 */
 static void pcache1FreePage(PgHdr1 *p){
   if( ALWAYS(p) ){
-    if( p->pCache->bPurgeable ){
-      pcache1.nCurrentPage--;
+    PCache1 *pCache = p->pCache;
+    if( pCache->bPurgeable ){
+      pCache->pGroup->nCurrentPage--;
     }
     pcache1Free(PGHDR1_TO_PAGE(p));
   }
@@ -33582,20 +33725,14 @@ static void pcache1FreePage(PgHdr1 *p){
 ** exists, this function falls back to sqlite3Malloc().
 */
 SQLITE_PRIVATE void *sqlite3PageMalloc(int sz){
-  void *p;
-  pcache1EnterMutex();
-  p = pcache1Alloc(sz);
-  pcache1LeaveMutex();
-  return p;
+  return pcache1Alloc(sz);
 }
 
 /*
 ** Free an allocated buffer obtained from sqlite3PageMalloc().
 */
 SQLITE_PRIVATE void sqlite3PageFree(void *p){
-  pcache1EnterMutex();
   pcache1Free(p);
-  pcache1LeaveMutex();
 }
 
 
@@ -33616,9 +33753,8 @@ SQLITE_PRIVATE void sqlite3PageFree(void *p){
 ** the heap even further.
 */
 static int pcache1UnderMemoryPressure(PCache1 *pCache){
-  assert( sqlite3_mutex_held(pcache1.mutex) );
   if( pcache1.nSlot && pCache->szPage<=pcache1.szSlot ){
-    return pcache1.nFreeSlot<pcache1.nReserve;
+    return pcache1.bUnderPressure;
   }else{
     return sqlite3HeapNearlyFull();
   }
@@ -33631,25 +33767,25 @@ static int pcache1UnderMemoryPressure(PCache1 *pCache){
 ** This function is used to resize the hash table used by the cache passed
 ** as the first argument.
 **
-** The global mutex must be held when this function is called.
+** The PCache mutex must be held when this function is called.
 */
 static int pcache1ResizeHash(PCache1 *p){
   PgHdr1 **apNew;
   unsigned int nNew;
   unsigned int i;
 
-  assert( sqlite3_mutex_held(pcache1.mutex) );
+  assert( sqlite3_mutex_held(p->pGroup->mutex) );
 
   nNew = p->nHash*2;
   if( nNew<256 ){
     nNew = 256;
   }
 
-  pcache1LeaveMutex();
+  pcache1LeaveMutex(p->pGroup);
   if( p->nHash ){ sqlite3BeginBenignMalloc(); }
   apNew = (PgHdr1 **)sqlite3_malloc(sizeof(PgHdr1 *)*nNew);
   if( p->nHash ){ sqlite3EndBenignMalloc(); }
-  pcache1EnterMutex();
+  pcache1EnterMutex(p->pGroup);
   if( apNew ){
     memset(apNew, 0, sizeof(PgHdr1 *)*nNew);
     for(i=0; i<p->nHash; i++){
@@ -33672,25 +33808,33 @@ static int pcache1ResizeHash(PCache1 *p){
 
 /*
 ** This function is used internally to remove the page pPage from the 
-** global LRU list, if is part of it. If pPage is not part of the global
+** PGroup LRU list, if is part of it. If pPage is not part of the PGroup
 ** LRU list, then this function is a no-op.
 **
-** The global mutex must be held when this function is called.
+** The PGroup mutex must be held when this function is called.
+**
+** If pPage is NULL then this routine is a no-op.
 */
 static void pcache1PinPage(PgHdr1 *pPage){
-  assert( sqlite3_mutex_held(pcache1.mutex) );
-  if( pPage && (pPage->pLruNext || pPage==pcache1.pLruTail) ){
+  PCache1 *pCache;
+  PGroup *pGroup;
+
+  if( pPage==0 ) return;
+  pCache = pPage->pCache;
+  pGroup = pCache->pGroup;
+  assert( sqlite3_mutex_held(pGroup->mutex) );
+  if( pPage->pLruNext || pPage==pGroup->pLruTail ){
     if( pPage->pLruPrev ){
       pPage->pLruPrev->pLruNext = pPage->pLruNext;
     }
     if( pPage->pLruNext ){
       pPage->pLruNext->pLruPrev = pPage->pLruPrev;
     }
-    if( pcache1.pLruHead==pPage ){
-      pcache1.pLruHead = pPage->pLruNext;
+    if( pGroup->pLruHead==pPage ){
+      pGroup->pLruHead = pPage->pLruNext;
     }
-    if( pcache1.pLruTail==pPage ){
-      pcache1.pLruTail = pPage->pLruPrev;
+    if( pGroup->pLruTail==pPage ){
+      pGroup->pLruTail = pPage->pLruPrev;
     }
     pPage->pLruNext = 0;
     pPage->pLruPrev = 0;
@@ -33703,13 +33847,14 @@ static void pcache1PinPage(PgHdr1 *pPage){
 ** Remove the page supplied as an argument from the hash table 
 ** (PCache1.apHash structure) that it is currently stored in.
 **
-** The global mutex must be held when this function is called.
+** The PGroup mutex must be held when this function is called.
 */
 static void pcache1RemoveFromHash(PgHdr1 *pPage){
   unsigned int h;
   PCache1 *pCache = pPage->pCache;
   PgHdr1 **pp;
 
+  assert( sqlite3_mutex_held(pCache->pGroup->mutex) );
   h = pPage->iKey % pCache->nHash;
   for(pp=&pCache->apHash[h]; (*pp)!=pPage; pp=&(*pp)->pNext);
   *pp = (*pp)->pNext;
@@ -33718,13 +33863,14 @@ static void pcache1RemoveFromHash(PgHdr1 *pPage){
 }
 
 /*
-** If there are currently more than pcache.nMaxPage pages allocated, try
-** to recycle pages to reduce the number allocated to pcache.nMaxPage.
+** If there are currently more than nMaxPage pages allocated, try
+** to recycle pages to reduce the number allocated to nMaxPage.
 */
-static void pcache1EnforceMaxPage(void){
-  assert( sqlite3_mutex_held(pcache1.mutex) );
-  while( pcache1.nCurrentPage>pcache1.nMaxPage && pcache1.pLruTail ){
-    PgHdr1 *p = pcache1.pLruTail;
+static void pcache1EnforceMaxPage(PGroup *pGroup){
+  assert( sqlite3_mutex_held(pGroup->mutex) );
+  while( pGroup->nCurrentPage>pGroup->nMaxPage && pGroup->pLruTail ){
+    PgHdr1 *p = pGroup->pLruTail;
+    assert( p->pCache->pGroup==pGroup );
     pcache1PinPage(p);
     pcache1RemoveFromHash(p);
     pcache1FreePage(p);
@@ -33736,15 +33882,15 @@ static void pcache1EnforceMaxPage(void){
 ** greater than or equal to iLimit. Any pinned pages that meet this 
 ** criteria are unpinned before they are discarded.
 **
-** The global mutex must be held when this function is called.
+** The PCache mutex must be held when this function is called.
 */
 static void pcache1TruncateUnsafe(
-  PCache1 *pCache, 
-  unsigned int iLimit 
+  PCache1 *pCache,             /* The cache to truncate */
+  unsigned int iLimit          /* Drop pages with this pgno or larger */
 ){
-  TESTONLY( unsigned int nPage = 0; )      /* Used to assert pCache->nPage is correct */
+  TESTONLY( unsigned int nPage = 0; )  /* To assert pCache->nPage is correct */
   unsigned int h;
-  assert( sqlite3_mutex_held(pcache1.mutex) );
+  assert( sqlite3_mutex_held(pCache->pGroup->mutex) );
   for(h=0; h<pCache->nHash; h++){
     PgHdr1 **pp = &pCache->apHash[h]; 
     PgHdr1 *pPage;
@@ -33774,8 +33920,10 @@ static int pcache1Init(void *NotUsed){
   assert( pcache1.isInit==0 );
   memset(&pcache1, 0, sizeof(pcache1));
   if( sqlite3GlobalConfig.bCoreMutex ){
-    pcache1.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU);
+    pcache1.grp.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU);
+    pcache1.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_PMEM);
   }
+  pcache1.grp.mxPinned = 10;
   pcache1.isInit = 1;
   return SQLITE_OK;
 }
@@ -33797,18 +33945,47 @@ static void pcache1Shutdown(void *NotUsed){
 ** Allocate a new cache.
 */
 static sqlite3_pcache *pcache1Create(int szPage, int bPurgeable){
-  PCache1 *pCache;
+  PCache1 *pCache;      /* The newly created page cache */
+  PGroup *pGroup;       /* The group the new page cache will belong to */
+  int sz;               /* Bytes of memory required to allocate the new cache */
 
-  pCache = (PCache1 *)sqlite3_malloc(sizeof(PCache1));
+  /*
+  ** The seperateCache variable is true if each PCache has its own private
+  ** PGroup.  In other words, separateCache is true for mode (1) where no
+  ** mutexing is required.
+  **
+  **   *  Always use a unified cache (mode-2) if ENABLE_MEMORY_MANAGEMENT
+  **
+  **   *  Always use a unified cache in single-threaded applications
+  **
+  **   *  Otherwise (if multi-threaded and ENABLE_MEMORY_MANAGEMENT is off)
+  **      use separate caches (mode-1)
+  */
+#if defined(SQLITE_ENABLE_MEMORY_MANAGEMENT) || SQLITE_THREADSAFE==0
+  const int separateCache = 0;
+#else
+  int separateCache = sqlite3GlobalConfig.bCoreMutex>0;
+#endif
+
+  sz = sizeof(PCache1) + sizeof(PGroup)*separateCache;
+  pCache = (PCache1 *)sqlite3_malloc(sz);
   if( pCache ){
-    memset(pCache, 0, sizeof(PCache1));
+    memset(pCache, 0, sz);
+    if( separateCache ){
+      pGroup = (PGroup*)&pCache[1];
+      pGroup->mxPinned = 10;
+    }else{
+      pGroup = &pcache1_g.grp;
+    }
+    pCache->pGroup = pGroup;
     pCache->szPage = szPage;
     pCache->bPurgeable = (bPurgeable ? 1 : 0);
     if( bPurgeable ){
       pCache->nMin = 10;
-      pcache1EnterMutex();
-      pcache1.nMinPage += pCache->nMin;
-      pcache1LeaveMutex();
+      pcache1EnterMutex(pGroup);
+      pGroup->nMinPage += pCache->nMin;
+      pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage;
+      pcache1LeaveMutex(pGroup);
     }
   }
   return (sqlite3_pcache *)pCache;
@@ -33822,11 +33999,14 @@ static sqlite3_pcache *pcache1Create(int szPage, int bPurgeable){
 static void pcache1Cachesize(sqlite3_pcache *p, int nMax){
   PCache1 *pCache = (PCache1 *)p;
   if( pCache->bPurgeable ){
-    pcache1EnterMutex();
-    pcache1.nMaxPage += (nMax - pCache->nMax);
+    PGroup *pGroup = pCache->pGroup;
+    pcache1EnterMutex(pGroup);
+    pGroup->nMaxPage += (nMax - pCache->nMax);
+    pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage;
     pCache->nMax = nMax;
-    pcache1EnforceMaxPage();
-    pcache1LeaveMutex();
+    pCache->n90pct = pCache->nMax*9/10;
+    pcache1EnforceMaxPage(pGroup);
+    pcache1LeaveMutex(pGroup);
   }
 }
 
@@ -33835,9 +34015,10 @@ static void pcache1Cachesize(sqlite3_pcache *p, int nMax){
 */
 static int pcache1Pagecount(sqlite3_pcache *p){
   int n;
-  pcache1EnterMutex();
-  n = ((PCache1 *)p)->nPage;
-  pcache1LeaveMutex();
+  PCache1 *pCache = (PCache1*)p;
+  pcache1EnterMutex(pCache->pGroup);
+  n = pCache->nPage;
+  pcache1LeaveMutex(pCache->pGroup);
   return n;
 }
 
@@ -33896,30 +34077,49 @@ static int pcache1Pagecount(sqlite3_pcache *p){
 **   5. Otherwise, allocate and return a new page buffer.
 */
 static void *pcache1Fetch(sqlite3_pcache *p, unsigned int iKey, int createFlag){
-  unsigned int nPinned;
+  int nPinned;
   PCache1 *pCache = (PCache1 *)p;
+  PGroup *pGroup;
   PgHdr1 *pPage = 0;
 
   assert( pCache->bPurgeable || createFlag!=1 );
-  pcache1EnterMutex();
-  if( createFlag==1 ) sqlite3BeginBenignMalloc();
+  assert( pCache->bPurgeable || pCache->nMin==0 );
+  assert( pCache->bPurgeable==0 || pCache->nMin==10 );
+  assert( pCache->nMin==0 || pCache->bPurgeable );
+  pcache1EnterMutex(pGroup = pCache->pGroup);
 
-  /* Search the hash table for an existing entry. */
+  /* Step 1: Search the hash table for an existing entry. */
   if( pCache->nHash>0 ){
     unsigned int h = iKey % pCache->nHash;
     for(pPage=pCache->apHash[h]; pPage&&pPage->iKey!=iKey; pPage=pPage->pNext);
   }
 
+  /* Step 2: Abort if no existing page is found and createFlag is 0 */
   if( pPage || createFlag==0 ){
     pcache1PinPage(pPage);
     goto fetch_out;
   }
 
-  /* Step 3 of header comment. */
+  /* The pGroup local variable will normally be initialized by the
+  ** pcache1EnterMutex() macro above.  But if SQLITE_MUTEX_OMIT is defined,
+  ** then pcache1EnterMutex() is a no-op, so we have to initialize the
+  ** local variable here.  Delaying the initialization of pGroup is an
+  ** optimization:  The common case is to exit the module before reaching
+  ** this point.
+  */
+#ifdef SQLITE_MUTEX_OMIT
+  pGroup = pCache->pGroup;
+#endif
+
+
+  /* Step 3: Abort if createFlag is 1 but the cache is nearly full */
   nPinned = pCache->nPage - pCache->nRecyclable;
+  assert( nPinned>=0 );
+  assert( pGroup->mxPinned == pGroup->nMaxPage + 10 - pGroup->nMinPage );
+  assert( pCache->n90pct == pCache->nMax*9/10 );
   if( createFlag==1 && (
-        nPinned>=(pcache1.nMaxPage+pCache->nMin-pcache1.nMinPage)
-     || nPinned>=(pCache->nMax * 9 / 10)
+        nPinned>=pGroup->mxPinned
+     || nPinned>=(int)pCache->n90pct
      || pcache1UnderMemoryPressure(pCache)
   )){
     goto fetch_out;
@@ -33929,20 +34129,22 @@ static void *pcache1Fetch(sqlite3_pcache *p, unsigned int iKey, int createFlag){
     goto fetch_out;
   }
 
-  /* Step 4. Try to recycle a page buffer if appropriate. */
-  if( pCache->bPurgeable && pcache1.pLruTail && (
+  /* Step 4. Try to recycle a page. */
+  if( pCache->bPurgeable && pGroup->pLruTail && (
          (pCache->nPage+1>=pCache->nMax)
-      || pcache1.nCurrentPage>=pcache1.nMaxPage
+      || pGroup->nCurrentPage>=pGroup->nMaxPage
       || pcache1UnderMemoryPressure(pCache)
   )){
-    pPage = pcache1.pLruTail;
+    PCache1 *pOtherCache;
+    pPage = pGroup->pLruTail;
     pcache1RemoveFromHash(pPage);
     pcache1PinPage(pPage);
-    if( pPage->pCache->szPage!=pCache->szPage ){
+    if( (pOtherCache = pPage->pCache)->szPage!=pCache->szPage ){
       pcache1FreePage(pPage);
       pPage = 0;
     }else{
-      pcache1.nCurrentPage -= (pPage->pCache->bPurgeable - pCache->bPurgeable);
+      pGroup->nCurrentPage -= 
+               (pOtherCache->bPurgeable - pCache->bPurgeable);
     }
   }
 
@@ -33950,7 +34152,11 @@ static void *pcache1Fetch(sqlite3_pcache *p, unsigned int iKey, int createFlag){
   ** attempt to allocate a new one. 
   */
   if( !pPage ){
+    if( createFlag==1 ) sqlite3BeginBenignMalloc();
+    pcache1LeaveMutex(pGroup);
     pPage = pcache1AllocPage(pCache);
+    pcache1EnterMutex(pGroup);
+    if( createFlag==1 ) sqlite3EndBenignMalloc();
   }
 
   if( pPage ){
@@ -33969,8 +34175,7 @@ fetch_out:
   if( pPage && iKey>pCache->iMaxKey ){
     pCache->iMaxKey = iKey;
   }
-  if( createFlag==1 ) sqlite3EndBenignMalloc();
-  pcache1LeaveMutex();
+  pcache1LeaveMutex(pGroup);
   return (pPage ? PGHDR1_TO_PAGE(pPage) : 0);
 }
 
@@ -33983,37 +34188,34 @@ fetch_out:
 static void pcache1Unpin(sqlite3_pcache *p, void *pPg, int reuseUnlikely){
   PCache1 *pCache = (PCache1 *)p;
   PgHdr1 *pPage = PAGE_TO_PGHDR1(pCache, pPg);
+  PGroup *pGroup = pCache->pGroup;
  
   assert( pPage->pCache==pCache );
-  pcache1EnterMutex();
+  pcache1EnterMutex(pGroup);
 
   /* It is an error to call this function if the page is already 
-  ** part of the global LRU list.
+  ** part of the PGroup LRU list.
   */
   assert( pPage->pLruPrev==0 && pPage->pLruNext==0 );
-  assert( pcache1.pLruHead!=pPage && pcache1.pLruTail!=pPage );
+  assert( pGroup->pLruHead!=pPage && pGroup->pLruTail!=pPage );
 
-  if( reuseUnlikely || pcache1.nCurrentPage>pcache1.nMaxPage ){
+  if( reuseUnlikely || pGroup->nCurrentPage>pGroup->nMaxPage ){
     pcache1RemoveFromHash(pPage);
     pcache1FreePage(pPage);
   }else{
-    /* Add the page to the global LRU list. Normally, the page is added to
-    ** the head of the list (last page to be recycled). However, if the 
-    ** reuseUnlikely flag passed to this function is true, the page is added
-    ** to the tail of the list (first page to be recycled).
-    */
-    if( pcache1.pLruHead ){
-      pcache1.pLruHead->pLruPrev = pPage;
-      pPage->pLruNext = pcache1.pLruHead;
-      pcache1.pLruHead = pPage;
+    /* Add the page to the PGroup LRU list. */
+    if( pGroup->pLruHead ){
+      pGroup->pLruHead->pLruPrev = pPage;
+      pPage->pLruNext = pGroup->pLruHead;
+      pGroup->pLruHead = pPage;
     }else{
-      pcache1.pLruTail = pPage;
-      pcache1.pLruHead = pPage;
+      pGroup->pLruTail = pPage;
+      pGroup->pLruHead = pPage;
     }
     pCache->nRecyclable++;
   }
 
-  pcache1LeaveMutex();
+  pcache1LeaveMutex(pCache->pGroup);
 }
 
 /*
@@ -34032,7 +34234,7 @@ static void pcache1Rekey(
   assert( pPage->iKey==iOld );
   assert( pPage->pCache==pCache );
 
-  pcache1EnterMutex();
+  pcache1EnterMutex(pCache->pGroup);
 
   h = iOld%pCache->nHash;
   pp = &pCache->apHash[h];
@@ -34049,7 +34251,7 @@ static void pcache1Rekey(
     pCache->iMaxKey = iNew;
   }
 
-  pcache1LeaveMutex();
+  pcache1LeaveMutex(pCache->pGroup);
 }
 
 /*
@@ -34061,12 +34263,12 @@ static void pcache1Rekey(
 */
 static void pcache1Truncate(sqlite3_pcache *p, unsigned int iLimit){
   PCache1 *pCache = (PCache1 *)p;
-  pcache1EnterMutex();
+  pcache1EnterMutex(pCache->pGroup);
   if( iLimit<=pCache->iMaxKey ){
     pcache1TruncateUnsafe(pCache, iLimit);
     pCache->iMaxKey = iLimit-1;
   }
-  pcache1LeaveMutex();
+  pcache1LeaveMutex(pCache->pGroup);
 }
 
 /*
@@ -34076,13 +34278,15 @@ static void pcache1Truncate(sqlite3_pcache *p, unsigned int iLimit){
 */
 static void pcache1Destroy(sqlite3_pcache *p){
   PCache1 *pCache = (PCache1 *)p;
+  PGroup *pGroup = pCache->pGroup;
   assert( pCache->bPurgeable || (pCache->nMax==0 && pCache->nMin==0) );
-  pcache1EnterMutex();
+  pcache1EnterMutex(pGroup);
   pcache1TruncateUnsafe(pCache, 0);
-  pcache1.nMaxPage -= pCache->nMax;
-  pcache1.nMinPage -= pCache->nMin;
-  pcache1EnforceMaxPage();
-  pcache1LeaveMutex();
+  pGroup->nMaxPage -= pCache->nMax;
+  pGroup->nMinPage -= pCache->nMin;
+  pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage;
+  pcache1EnforceMaxPage(pGroup);
+  pcache1LeaveMutex(pGroup);
   sqlite3_free(pCache->apHash);
   sqlite3_free(pCache);
 }
@@ -34121,16 +34325,18 @@ SQLITE_PRIVATE void sqlite3PCacheSetDefault(void){
 */
 SQLITE_PRIVATE int sqlite3PcacheReleaseMemory(int nReq){
   int nFree = 0;
+  assert( sqlite3_mutex_notheld(pcache1.grp.mutex) );
+  assert( sqlite3_mutex_notheld(pcache1.mutex) );
   if( pcache1.pStart==0 ){
     PgHdr1 *p;
-    pcache1EnterMutex();
-    while( (nReq<0 || nFree<nReq) && ((p=pcache1.pLruTail)!=0) ){
+    pcache1EnterMutex(&pcache1.grp);
+    while( (nReq<0 || nFree<nReq) && ((p=pcache1.grp.pLruTail)!=0) ){
       nFree += pcache1MemSize(PGHDR1_TO_PAGE(p));
       pcache1PinPage(p);
       pcache1RemoveFromHash(p);
       pcache1FreePage(p);
     }
-    pcache1LeaveMutex();
+    pcache1LeaveMutex(&pcache1.grp);
   }
   return nFree;
 }
@@ -34149,12 +34355,12 @@ SQLITE_PRIVATE void sqlite3PcacheStats(
 ){
   PgHdr1 *p;
   int nRecyclable = 0;
-  for(p=pcache1.pLruHead; p; p=p->pLruNext){
+  for(p=pcache1.grp.pLruHead; p; p=p->pLruNext){
     nRecyclable++;
   }
-  *pnCurrent = pcache1.nCurrentPage;
-  *pnMax = pcache1.nMaxPage;
-  *pnMin = pcache1.nMinPage;
+  *pnCurrent = pcache1.grp.nCurrentPage;
+  *pnMax = pcache1.grp.nMaxPage;
+  *pnMin = pcache1.grp.nMinPage;
   *pnRecyclable = nRecyclable;
 }
 #endif
@@ -37179,15 +37385,21 @@ static int pager_truncate(Pager *pPager, Pgno nPage){
    && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) 
   ){
     i64 currentSize, newSize;
+    int szPage = pPager->pageSize;
     assert( pPager->eLock==EXCLUSIVE_LOCK );
     /* TODO: Is it safe to use Pager.dbFileSize here? */
     rc = sqlite3OsFileSize(pPager->fd, &currentSize);
-    newSize = pPager->pageSize*(i64)nPage;
+    newSize = szPage*(i64)nPage;
     if( rc==SQLITE_OK && currentSize!=newSize ){
       if( currentSize>newSize ){
         rc = sqlite3OsTruncate(pPager->fd, newSize);
       }else{
-        rc = sqlite3OsWrite(pPager->fd, "", 1, newSize-1);
+        char *pTmp = pPager->pTmpSpace;
+        memset(pTmp, 0, szPage);
+        testcase( (newSize-szPage) <  currentSize );
+        testcase( (newSize-szPage) == currentSize );
+        testcase( (newSize-szPage) >  currentSize );
+        rc = sqlite3OsWrite(pPager->fd, pTmp, szPage, newSize-szPage);
       }
       if( rc==SQLITE_OK ){
         pPager->dbFileSize = nPage;
@@ -37451,10 +37663,10 @@ end_playback:
     rc = readMasterJournal(pPager->jfd, zMaster, pPager->pVfs->mxPathname+1);
     testcase( rc!=SQLITE_OK );
   }
-  if( rc==SQLITE_OK && !pPager->noSync 
+  if( rc==SQLITE_OK
    && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN)
   ){
-    rc = sqlite3OsSync(pPager->fd, pPager->syncFlags);
+    rc = sqlite3PagerSync(pPager);
   }
   if( rc==SQLITE_OK ){
     rc = pager_end_transaction(pPager, zMaster[0]!='\0');
@@ -37617,11 +37829,37 @@ static int pagerRollbackWal(Pager *pPager){
   return rc;
 }
 
+
+/*
+** Update the value of the change-counter at offsets 24 and 92 in
+** the header and the sqlite version number at offset 96.
+**
+** This is an unconditional update.  See also the pager_incr_changecounter()
+** routine which only updates the change-counter if the update is actually
+** needed, as determined by the pPager->changeCountDone state variable.
+*/
+static void pager_write_changecounter(PgHdr *pPg){
+  u32 change_counter;
+
+  /* Increment the value just read and write it back to byte 24. */
+  change_counter = sqlite3Get4byte((u8*)pPg->pPager->dbFileVers)+1;
+  put32bits(((char*)pPg->pData)+24, change_counter);
+
+  /* Also store the SQLite version number in bytes 96..99 and in
+  ** bytes 92..95 store the change counter for which the version number
+  ** is valid. */
+  put32bits(((char*)pPg->pData)+92, change_counter);
+  put32bits(((char*)pPg->pData)+96, SQLITE_VERSION_NUMBER);
+}
+
 /*
 ** This function is a wrapper around sqlite3WalFrames(). As well as logging
 ** the contents of the list of pages headed by pList (connected by pDirty),
 ** this function notifies any active backup processes that the pages have
-** changed. 
+** changed.
+**
+** The list of pages passed into this routine is always sorted by page number.
+** Hence, if page 1 appears anywhere on the list, it will be the first page.
 */ 
 static int pagerWalFrames(
   Pager *pPager,                  /* Pager object */
@@ -37631,8 +37869,19 @@ static int pagerWalFrames(
   int syncFlags                   /* Flags to pass to OsSync() (or 0) */
 ){
   int rc;                         /* Return code */
+#if defined(SQLITE_DEBUG) || defined(SQLITE_CHECK_PAGES)
+  PgHdr *p;                       /* For looping over pages */
+#endif
 
   assert( pPager->pWal );
+#ifdef SQLITE_DEBUG
+  /* Verify that the page list is in accending order */
+  for(p=pList; p && p->pDirty; p=p->pDirty){
+    assert( p->pgno < p->pDirty->pgno );
+  }
+#endif
+
+  if( pList->pgno==1 ) pager_write_changecounter(pList);
   rc = sqlite3WalFrames(pPager->pWal, 
       pPager->pageSize, pList, nTruncate, isCommit, syncFlags
   );
@@ -37644,9 +37893,8 @@ static int pagerWalFrames(
   }
 
 #ifdef SQLITE_CHECK_PAGES
-  {
-    PgHdr *p;
-    for(p=pList; p; p=p->pDirty) pager_set_pagehash(p);
+  for(p=pList; p; p=p->pDirty){
+    pager_set_pagehash(p);
   }
 #endif
 
@@ -38671,6 +38919,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){
       char *pData;                                   /* Data to write */    
 
       assert( (pList->flags&PGHDR_NEED_SYNC)==0 );
+      if( pList->pgno==1 ) pager_write_changecounter(pList);
 
       /* Encode the database */
       CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData);
@@ -40191,7 +40440,13 @@ SQLITE_PRIVATE void sqlite3PagerDontWrite(PgHdr *pPg){
 /*
 ** This routine is called to increment the value of the database file 
 ** change-counter, stored as a 4-byte big-endian integer starting at 
-** byte offset 24 of the pager file.
+** byte offset 24 of the pager file.  The secondary change counter at
+** 92 is also updated, as is the SQLite version number at offset 96.
+**
+** But this only happens if the pPager->changeCountDone flag is false.
+** To avoid excess churning of page 1, the update only happens once.
+** See also the pager_write_changecounter() routine that does an 
+** unconditional update of the change counters.
 **
 ** If the isDirectMode flag is zero, then this is done by calling 
 ** sqlite3PagerWrite() on page 1, then modifying the contents of the
@@ -40232,7 +40487,6 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){
 
   if( !pPager->changeCountDone && pPager->dbSize>0 ){
     PgHdr *pPgHdr;                /* Reference to page 1 */
-    u32 change_counter;           /* Initial value of change-counter field */
 
     assert( !pPager->tempFile && isOpen(pPager->fd) );
 
@@ -40250,16 +40504,8 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){
     }
 
     if( rc==SQLITE_OK ){
-      /* Increment the value just read and write it back to byte 24. */
-      change_counter = sqlite3Get4byte((u8*)pPager->dbFileVers);
-      change_counter++;
-      put32bits(((char*)pPgHdr->pData)+24, change_counter);
-
-      /* Also store the SQLite version number in bytes 96..99 and in
-      ** bytes 92..95 store the change counter for which the version number
-      ** is valid. */
-      put32bits(((char*)pPgHdr->pData)+92, change_counter);
-      put32bits(((char*)pPgHdr->pData)+96, SQLITE_VERSION_NUMBER);
+      /* Actually do the update of the change counter */
+      pager_write_changecounter(pPgHdr);
 
       /* If running in direct mode, write the contents of page 1 to the file. */
       if( DIRECT_MODE ){
@@ -40291,12 +40537,13 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){
 ** function returns SQLITE_OK. Otherwise, an IO error code is returned.
 */
 SQLITE_PRIVATE int sqlite3PagerSync(Pager *pPager){
-  int rc;                              /* Return code */
-  assert( !MEMDB );
-  if( pPager->noSync ){
-    rc = SQLITE_OK;
-  }else{
+  int rc = SQLITE_OK;
+  if( !pPager->noSync ){
+    assert( !MEMDB );
     rc = sqlite3OsSync(pPager->fd, pPager->syncFlags);
+  }else if( isOpen(pPager->fd) ){
+    assert( !MEMDB );
+    sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_SYNC_OMITTED, (void *)&rc);
   }
   return rc;
 }
@@ -40515,8 +40762,8 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne(
       }
   
       /* Finally, sync the database file. */
-      if( !pPager->noSync && !noSync ){
-        rc = sqlite3OsSync(pPager->fd, pPager->syncFlags);
+      if( !noSync ){
+        rc = sqlite3PagerSync(pPager);
       }
       IOTRACE(("DBSYNC %p\n", pPager))
     }
@@ -40628,7 +40875,17 @@ SQLITE_PRIVATE int sqlite3PagerRollback(Pager *pPager){
     rc2 = pager_end_transaction(pPager, pPager->setMaster);
     if( rc==SQLITE_OK ) rc = rc2;
   }else if( !isOpen(pPager->jfd) || pPager->eState==PAGER_WRITER_LOCKED ){
+    int eState = pPager->eState;
     rc = pager_end_transaction(pPager, 0);
+    if( !MEMDB && eState>PAGER_WRITER_LOCKED ){
+      /* This can happen using journal_mode=off. Move the pager to the error 
+      ** state to indicate that the contents of the cache may not be trusted.
+      ** Any active readers will get SQLITE_ABORT.
+      */
+      pPager->errCode = SQLITE_ABORT;
+      pPager->eState = PAGER_ERROR;
+      return rc;
+    }
   }else{
     rc = pager_playback(pPager, 0);
   }
@@ -41898,14 +42155,14 @@ typedef u16 ht_slot;
 */
 struct WalIterator {
   int iPrior;                     /* Last result returned from the iterator */
-  int nSegment;                   /* Size of the aSegment[] array */
+  int nSegment;                   /* Number of entries in aSegment[] */
   struct WalSegment {
     int iNext;                    /* Next slot in aIndex[] not yet returned */
     ht_slot *aIndex;              /* i0, i1, i2... such that aPgno[iN] ascend */
     u32 *aPgno;                   /* Array of page numbers. */
-    int nEntry;                   /* Max size of aPgno[] and aIndex[] arrays */
+    int nEntry;                   /* Nr. of entries in aPgno[] and aIndex[] */
     int iZero;                    /* Frame number associated with aPgno[0] */
-  } aSegment[1];                  /* One for every 32KB page in the WAL */
+  } aSegment[1];                  /* One for every 32KB page in the wal-index */
 };
 
 /*
@@ -42769,9 +43026,29 @@ static int walIteratorNext(
 
 /*
 ** This function merges two sorted lists into a single sorted list.
+**
+** aLeft[] and aRight[] are arrays of indices.  The sort key is
+** aContent[aLeft[]] and aContent[aRight[]].  Upon entry, the following
+** is guaranteed for all J<K:
+**
+**        aContent[aLeft[J]] < aContent[aLeft[K]]
+**        aContent[aRight[J]] < aContent[aRight[K]]
+**
+** This routine overwrites aRight[] with a new (probably longer) sequence
+** of indices such that the aRight[] contains every index that appears in
+** either aLeft[] or the old aRight[] and such that the second condition
+** above is still met.
+**
+** The aContent[aLeft[X]] values will be unique for all X.  And the
+** aContent[aRight[X]] values will be unique too.  But there might be
+** one or more combinations of X and Y such that
+**
+**      aLeft[X]!=aRight[Y]  &&  aContent[aLeft[X]] == aContent[aRight[Y]]
+**
+** When that happens, omit the aLeft[X] and use the aRight[Y] index.
 */
 static void walMerge(
-  u32 *aContent,                  /* Pages in wal */
+  const u32 *aContent,            /* Pages in wal - keys for the sort */
   ht_slot *aLeft,                 /* IN: Left hand input list */
   int nLeft,                      /* IN: Elements in array *paLeft */
   ht_slot **paRight,              /* IN/OUT: Right hand input list */
@@ -42811,10 +43088,24 @@ static void walMerge(
 }
 
 /*
-** Sort the elements in list aList, removing any duplicates.
+** Sort the elements in list aList using aContent[] as the sort key.
+** Remove elements with duplicate keys, preferring to keep the
+** larger aList[] values.
+**
+** The aList[] entries are indices into aContent[].  The values in
+** aList[] are to be sorted so that for all J<K:
+**
+**      aContent[aList[J]] < aContent[aList[K]]
+**
+** For any X and Y such that
+**
+**      aContent[aList[X]] == aContent[aList[Y]]
+**
+** Keep the larger of the two values aList[X] and aList[Y] and discard
+** the smaller.
 */
 static void walMergesort(
-  u32 *aContent,                  /* Pages in wal */
+  const u32 *aContent,            /* Pages in wal */
   ht_slot *aBuffer,               /* Buffer of at least *pnList items to use */
   ht_slot *aList,                 /* IN/OUT: List to sort */
   int *pnList                     /* IN/OUT: Number of elements in aList[] */
@@ -42879,6 +43170,7 @@ static void walIteratorFree(WalIterator *p){
 /*
 ** Construct a WalInterator object that can be used to loop over all 
 ** pages in the WAL in ascending order. The caller must hold the checkpoint
+** lock.
 **
 ** On success, make *pp point to the newly allocated WalInterator object
 ** return SQLITE_OK. Otherwise, return an error code. If this routine
@@ -43013,7 +43305,8 @@ static int walCheckpoint(
   szPage = (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16);
   testcase( szPage<=32768 );
   testcase( szPage>=65536 );
-  if( pWal->hdr.mxFrame==0 ) return SQLITE_OK;
+  pInfo = walCkptInfo(pWal);
+  if( pInfo->nBackfill>=pWal->hdr.mxFrame ) return SQLITE_OK;
 
   /* Allocate the iterator */
   rc = walIteratorInit(pWal, &pIter);
@@ -43035,7 +43328,6 @@ static int walCheckpoint(
   */
   mxSafeFrame = pWal->hdr.mxFrame;
   mxPage = pWal->hdr.nPage;
-  pInfo = walCkptInfo(pWal);
   for(i=1; i<WAL_NREADER; i++){
     u32 y = pInfo->aReadMark[i];
     if( mxSafeFrame>=y ){
@@ -44625,12 +44917,12 @@ struct BtShared {
   u8 autoVacuum;        /* True if auto-vacuum is enabled */
   u8 incrVacuum;        /* True if incr-vacuum is enabled */
 #endif
+  u8 inTransaction;     /* Transaction state */
+  u8 doNotUseWAL;       /* If true, do not open write-ahead-log file */
   u16 maxLocal;         /* Maximum local payload in non-LEAFDATA tables */
   u16 minLocal;         /* Minimum local payload in non-LEAFDATA tables */
   u16 maxLeaf;          /* Maximum local payload in a LEAFDATA table */
   u16 minLeaf;          /* Minimum local payload in a LEAFDATA table */
-  u8 inTransaction;     /* Transaction state */
-  u8 doNotUseWAL;       /* If true, do not open write-ahead-log file */
   u32 pageSize;         /* Total number of bytes on a page */
   u32 usableSize;       /* Number of usable bytes on each page */
   int nTransaction;     /* Number of open transactions (read + write) */
@@ -44657,8 +44949,8 @@ struct BtShared {
 */
 typedef struct CellInfo CellInfo;
 struct CellInfo {
-  u8 *pCell;     /* Pointer to the start of cell content */
   i64 nKey;      /* The key for INTKEY tables, or number of bytes in key */
+  u8 *pCell;     /* Pointer to the start of cell content */
   u32 nData;     /* Number of bytes of data */
   u32 nPayload;  /* Total amount of payload */
   u16 nHeader;   /* Size of the cell content header in bytes */
@@ -44700,20 +44992,20 @@ struct BtCursor {
   Pgno pgnoRoot;            /* The root page of this tree */
   sqlite3_int64 cachedRowid; /* Next rowid cache.  0 means not valid */
   CellInfo info;            /* A parse of the cell we are pointing at */
+  i64 nKey;        /* Size of pKey, or last integer key */
+  void *pKey;      /* Saved key that was cursor's last known position */
+  int skipNext;    /* Prev() is noop if negative. Next() is noop if positive */
   u8 wrFlag;                /* True if writable */
   u8 atLast;                /* Cursor pointing to the last entry */
   u8 validNKey;             /* True if info.nKey is valid */
   u8 eState;                /* One of the CURSOR_XXX constants (see below) */
-  void *pKey;      /* Saved key that was cursor's last known position */
-  i64 nKey;        /* Size of pKey, or last integer key */
-  int skipNext;    /* Prev() is noop if negative. Next() is noop if positive */
 #ifndef SQLITE_OMIT_INCRBLOB
-  u8 isIncrblobHandle;      /* True if this cursor is an incr. io handle */
   Pgno *aOverflow;          /* Cache of overflow page locations */
+  u8 isIncrblobHandle;      /* True if this cursor is an incr. io handle */
 #endif
   i16 iPage;                            /* Index of current page in apPage */
-  MemPage *apPage[BTCURSOR_MAX_DEPTH];  /* Pages from root to current page */
   u16 aiIdx[BTCURSOR_MAX_DEPTH];        /* Current index in apPage[i] */
+  MemPage *apPage[BTCURSOR_MAX_DEPTH];  /* Pages from root to current page */
 };
 
 /*
@@ -46117,14 +46409,9 @@ static void btreeParseCellPtr(
     /* This is the (easy) common case where the entire payload fits
     ** on the local page.  No overflow is required.
     */
-    int nSize;          /* Total size of cell content in bytes */
-    nSize = nPayload + n;
+    if( (pInfo->nSize = (u16)(n+nPayload))<4 ) pInfo->nSize = 4;
     pInfo->nLocal = (u16)nPayload;
     pInfo->iOverflow = 0;
-    if( (nSize & ~3)==0 ){
-      nSize = 4;        /* Minimum cell size is 4 */
-    }
-    pInfo->nSize = (u16)nSize;
   }else{
     /* If the payload will not fit completely on the local page, we have
     ** to decide how much to store locally and how much to spill onto
@@ -47585,7 +47872,7 @@ static int lockBtree(BtShared *pBt){
                                    pageSize-usableSize);
       return rc;
     }
-    if( nPageHeader>nPageFile ){
+    if( (pBt->db->flags & SQLITE_RecoveryMode)==0 && nPageHeader>nPageFile ){
       rc = SQLITE_CORRUPT_BKPT;
       goto page1_init_failed;
     }
@@ -53457,6 +53744,16 @@ static Btree *findBtree(sqlite3 *pErrorDb, sqlite3 *pDb, const char *zDb){
   return pDb->aDb[i].pBt;
 }
 
+/*
+** Attempt to set the page size of the destination to match the page size
+** of the source.
+*/
+static int setDestPgsz(sqlite3_backup *p){
+  int rc;
+  rc = sqlite3BtreeSetPageSize(p->pDest,sqlite3BtreeGetPageSize(p->pSrc),-1,0);
+  return rc;
+}
+
 /*
 ** Create an sqlite3_backup process to copy the contents of zSrcDb from
 ** connection handle pSrcDb to zDestDb in pDestDb. If successful, return
@@ -53510,10 +53807,11 @@ SQLITE_API sqlite3_backup *sqlite3_backup_init(
     p->iNext = 1;
     p->isAttached = 0;
 
-    if( 0==p->pSrc || 0==p->pDest ){
-      /* One (or both) of the named databases did not exist. An error has
-      ** already been written into the pDestDb handle. All that is left
-      ** to do here is free the sqlite3_backup structure.
+    if( 0==p->pSrc || 0==p->pDest || setDestPgsz(p)==SQLITE_NOMEM ){
+      /* One (or both) of the named databases did not exist or an OOM
+      ** error was hit.  The error has already been written into the
+      ** pDestDb handle.  All that is left to do here is free the
+      ** sqlite3_backup structure.
       */
       sqlite3_free(p);
       p = 0;
@@ -53770,32 +54068,46 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){
         */
         const i64 iSize = (i64)pgszSrc * (i64)nSrcPage;
         sqlite3_file * const pFile = sqlite3PagerFile(pDestPager);
+        i64 iOff;
+        i64 iEnd;
 
         assert( pFile );
         assert( (i64)nDestTruncate*(i64)pgszDest >= iSize || (
               nDestTruncate==(int)(PENDING_BYTE_PAGE(p->pDest->pBt)-1)
            && iSize>=PENDING_BYTE && iSize<=PENDING_BYTE+pgszDest
         ));
-        if( SQLITE_OK==(rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 1))
-         && SQLITE_OK==(rc = backupTruncateFile(pFile, iSize))
-         && SQLITE_OK==(rc = sqlite3PagerSync(pDestPager))
+
+        /* This call ensures that all data required to recreate the original
+        ** database has been stored in the journal for pDestPager and the
+        ** journal synced to disk. So at this point we may safely modify
+        ** the database file in any way, knowing that if a power failure
+        ** occurs, the original database will be reconstructed from the 
+        ** journal file.  */
+        rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 1);
+
+        /* Write the extra pages and truncate the database file as required. */
+        iEnd = MIN(PENDING_BYTE + pgszDest, iSize);
+        for(
+          iOff=PENDING_BYTE+pgszSrc; 
+          rc==SQLITE_OK && iOff<iEnd; 
+          iOff+=pgszSrc
         ){
-          i64 iOff;
-          i64 iEnd = MIN(PENDING_BYTE + pgszDest, iSize);
-          for(
-            iOff=PENDING_BYTE+pgszSrc; 
-            rc==SQLITE_OK && iOff<iEnd; 
-            iOff+=pgszSrc
-          ){
-            PgHdr *pSrcPg = 0;
-            const Pgno iSrcPg = (Pgno)((iOff/pgszSrc)+1);
-            rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg);
-            if( rc==SQLITE_OK ){
-              u8 *zData = sqlite3PagerGetData(pSrcPg);
-              rc = sqlite3OsWrite(pFile, zData, pgszSrc, iOff);
-            }
-            sqlite3PagerUnref(pSrcPg);
+          PgHdr *pSrcPg = 0;
+          const Pgno iSrcPg = (Pgno)((iOff/pgszSrc)+1);
+          rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg);
+          if( rc==SQLITE_OK ){
+            u8 *zData = sqlite3PagerGetData(pSrcPg);
+            rc = sqlite3OsWrite(pFile, zData, pgszSrc, iOff);
           }
+          sqlite3PagerUnref(pSrcPg);
+        }
+        if( rc==SQLITE_OK ){
+          rc = backupTruncateFile(pFile, iSize);
+        }
+
+        /* Sync the database file to disk. */
+        if( rc==SQLITE_OK ){
+          rc = sqlite3PagerSync(pDestPager);
         }
       }else{
         rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 0);
@@ -55558,7 +55870,7 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){
     pOp->opflags = sqlite3OpcodeProperty[opcode];
     if( opcode==OP_Function || opcode==OP_AggStep ){
       if( pOp->p5>nMaxArgs ) nMaxArgs = pOp->p5;
-    }else if( opcode==OP_Transaction && pOp->p2!=0 ){
+    }else if( (opcode==OP_Transaction && pOp->p2!=0) || opcode==OP_Vacuum ){
       p->readOnly = 0;
 #ifndef SQLITE_OMIT_VIRTUALTABLE
     }else if( opcode==OP_VUpdate ){
@@ -58655,11 +58967,30 @@ static int sqlite3Step(Vdbe *p){
   assert(p);
   if( p->magic!=VDBE_MAGIC_RUN ){
     /* We used to require that sqlite3_reset() be called before retrying
-    ** sqlite3_step() after any error.  But after 3.6.23, we changed this
-    ** so that sqlite3_reset() would be called automatically instead of
-    ** throwing the error.
+    ** sqlite3_step() after any error or after SQLITE_DONE.  But beginning
+    ** with version 3.7.0, we changed this so that sqlite3_reset() would
+    ** be called automatically instead of throwing the SQLITE_MISUSE error.
+    ** This "automatic-reset" change is not technically an incompatibility, 
+    ** since any application that receives an SQLITE_MISUSE is broken by
+    ** definition.
+    **
+    ** Nevertheless, some published applications that were originally written
+    ** for version 3.6.23 or earlier do in fact depend on SQLITE_MISUSE 
+    ** returns, and the so were broken by the automatic-reset change.  As a
+    ** a work-around, the SQLITE_OMIT_AUTORESET compile-time restores the
+    ** legacy behavior of returning SQLITE_MISUSE for cases where the 
+    ** previous sqlite3_step() returned something other than a SQLITE_LOCKED
+    ** or SQLITE_BUSY error.
     */
+#ifdef SQLITE_OMIT_AUTORESET
+    if( p->rc==SQLITE_BUSY || p->rc==SQLITE_LOCKED ){
+      sqlite3_reset((sqlite3_stmt*)p);
+    }else{
+      return SQLITE_MISUSE_BKPT;
+    }
+#else
     sqlite3_reset((sqlite3_stmt*)p);
+#endif
   }
 
   /* Check that malloc() has not failed. If it has, return early. */
@@ -58701,7 +59032,9 @@ static int sqlite3Step(Vdbe *p){
   }else
 #endif /* SQLITE_OMIT_EXPLAIN */
   {
+    db->vdbeExecCnt++;
     rc = sqlite3VdbeExec(p);
+    db->vdbeExecCnt--;
   }
 
 #ifndef SQLITE_OMIT_TRACE
@@ -58995,7 +59328,7 @@ static Mem *columnMem(sqlite3_stmt *pStmt, int i){
 #if defined(SQLITE_DEBUG) && defined(__GNUC__)
       __attribute__((aligned(8))) 
 #endif
-      = {{0}, (double)0, 0, "", 0, MEM_Null, SQLITE_NULL, 0, 0, 0 };
+      = {0, "", (double)0, {0}, 0, MEM_Null, SQLITE_NULL, 0, 0, 0 };
 
     if( pVm && ALWAYS(pVm->db) ){
       sqlite3_mutex_enter(pVm->db->mutex);
@@ -59658,9 +59991,12 @@ static int findNextHostParameter(const char *zSql, int *pnToken){
 }
 
 /*
-** Return a pointer to a string in memory obtained form sqlite3DbMalloc() which
-** holds a copy of zRawSql but with host parameters expanded to their
-** current bindings.
+** This function returns a pointer to a nul-terminated string in memory
+** obtained from sqlite3DbMalloc(). If sqlite3.vdbeExecCnt is 1, then the
+** string contains a copy of zRawSql but with host parameters expanded to 
+** their current bindings. Or, if sqlite3.vdbeExecCnt is greater than 1, 
+** then the returned string holds a copy of zRawSql with "-- " prepended
+** to each line of text.
 **
 ** The calling function is responsible for making sure the memory returned
 ** is eventually freed.
@@ -59691,63 +60027,72 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql(
   sqlite3StrAccumInit(&out, zBase, sizeof(zBase), 
                       db->aLimit[SQLITE_LIMIT_LENGTH]);
   out.db = db;
-  while( zRawSql[0] ){
-    n = findNextHostParameter(zRawSql, &nToken);
-    assert( n>0 );
-    sqlite3StrAccumAppend(&out, zRawSql, n);
-    zRawSql += n;
-    assert( zRawSql[0] || nToken==0 );
-    if( nToken==0 ) break;
-    if( zRawSql[0]=='?' ){
-      if( nToken>1 ){
-        assert( sqlite3Isdigit(zRawSql[1]) );
-        sqlite3GetInt32(&zRawSql[1], &idx);
+  if( db->vdbeExecCnt>1 ){
+    while( *zRawSql ){
+      const char *zStart = zRawSql;
+      while( *(zRawSql++)!='\n' && *zRawSql );
+      sqlite3StrAccumAppend(&out, "-- ", 3);
+      sqlite3StrAccumAppend(&out, zStart, zRawSql-zStart);
+    }
+  }else{
+    while( zRawSql[0] ){
+      n = findNextHostParameter(zRawSql, &nToken);
+      assert( n>0 );
+      sqlite3StrAccumAppend(&out, zRawSql, n);
+      zRawSql += n;
+      assert( zRawSql[0] || nToken==0 );
+      if( nToken==0 ) break;
+      if( zRawSql[0]=='?' ){
+        if( nToken>1 ){
+          assert( sqlite3Isdigit(zRawSql[1]) );
+          sqlite3GetInt32(&zRawSql[1], &idx);
+        }else{
+          idx = nextIndex;
+        }
       }else{
-        idx = nextIndex;
-      }
-    }else{
-      assert( zRawSql[0]==':' || zRawSql[0]=='$' || zRawSql[0]=='@' );
-      testcase( zRawSql[0]==':' );
-      testcase( zRawSql[0]=='$' );
-      testcase( zRawSql[0]=='@' );
-      idx = sqlite3VdbeParameterIndex(p, zRawSql, nToken);
-      assert( idx>0 );
-    }
-    zRawSql += nToken;
-    nextIndex = idx + 1;
-    assert( idx>0 && idx<=p->nVar );
-    pVar = &p->aVar[idx-1];
-    if( pVar->flags & MEM_Null ){
-      sqlite3StrAccumAppend(&out, "NULL", 4);
-    }else if( pVar->flags & MEM_Int ){
-      sqlite3XPrintf(&out, "%lld", pVar->u.i);
-    }else if( pVar->flags & MEM_Real ){
-      sqlite3XPrintf(&out, "%!.15g", pVar->r);
-    }else if( pVar->flags & MEM_Str ){
+        assert( zRawSql[0]==':' || zRawSql[0]=='$' || zRawSql[0]=='@' );
+        testcase( zRawSql[0]==':' );
+        testcase( zRawSql[0]=='$' );
+        testcase( zRawSql[0]=='@' );
+        idx = sqlite3VdbeParameterIndex(p, zRawSql, nToken);
+        assert( idx>0 );
+      }
+      zRawSql += nToken;
+      nextIndex = idx + 1;
+      assert( idx>0 && idx<=p->nVar );
+      pVar = &p->aVar[idx-1];
+      if( pVar->flags & MEM_Null ){
+        sqlite3StrAccumAppend(&out, "NULL", 4);
+      }else if( pVar->flags & MEM_Int ){
+        sqlite3XPrintf(&out, "%lld", pVar->u.i);
+      }else if( pVar->flags & MEM_Real ){
+        sqlite3XPrintf(&out, "%!.15g", pVar->r);
+      }else if( pVar->flags & MEM_Str ){
 #ifndef SQLITE_OMIT_UTF16
-      u8 enc = ENC(db);
-      if( enc!=SQLITE_UTF8 ){
-        Mem utf8;
-        memset(&utf8, 0, sizeof(utf8));
-        utf8.db = db;
-        sqlite3VdbeMemSetStr(&utf8, pVar->z, pVar->n, enc, SQLITE_STATIC);
-        sqlite3VdbeChangeEncoding(&utf8, SQLITE_UTF8);
-        sqlite3XPrintf(&out, "'%.*q'", utf8.n, utf8.z);
-        sqlite3VdbeMemRelease(&utf8);
-      }else
+        u8 enc = ENC(db);
+        if( enc!=SQLITE_UTF8 ){
+          Mem utf8;
+          memset(&utf8, 0, sizeof(utf8));
+          utf8.db = db;
+          sqlite3VdbeMemSetStr(&utf8, pVar->z, pVar->n, enc, SQLITE_STATIC);
+          sqlite3VdbeChangeEncoding(&utf8, SQLITE_UTF8);
+          sqlite3XPrintf(&out, "'%.*q'", utf8.n, utf8.z);
+          sqlite3VdbeMemRelease(&utf8);
+        }else
 #endif
-      {
-        sqlite3XPrintf(&out, "'%.*q'", pVar->n, pVar->z);
-      }
-    }else if( pVar->flags & MEM_Zero ){
-      sqlite3XPrintf(&out, "zeroblob(%d)", pVar->u.nZero);
-    }else{
-      assert( pVar->flags & MEM_Blob );
-      sqlite3StrAccumAppend(&out, "x'", 2);
-      for(i=0; i<pVar->n; i++){
-        sqlite3XPrintf(&out, "%02x", pVar->z[i]&0xff);
+        {
+          sqlite3XPrintf(&out, "'%.*q'", pVar->n, pVar->z);
+        }
+      }else if( pVar->flags & MEM_Zero ){
+        sqlite3XPrintf(&out, "zeroblob(%d)", pVar->u.nZero);
+      }else{
+        assert( pVar->flags & MEM_Blob );
+        sqlite3StrAccumAppend(&out, "x'", 2);
+        for(i=0; i<pVar->n; i++){
+          sqlite3XPrintf(&out, "%02x", pVar->z[i]&0xff);
+        }
+        sqlite3StrAccumAppend(&out, "'", 1);
       }
-      sqlite3StrAccumAppend(&out, "'", 1);
     }
   }
   return sqlite3StrAccumFinish(&out);
@@ -73442,9 +73787,10 @@ static void analyzeOneTable(
     sqlite3VdbeAddOp2(v, OP_AddImm, iMem, 1);
 
     for(i=0; i<nCol; i++){
+      CollSeq *pColl;
       sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, regCol);
-#ifdef SQLITE_ENABLE_STAT2
       if( i==0 ){
+#ifdef SQLITE_ENABLE_STAT2
         /* Check if the record that cursor iIdxCur points to contains a
         ** value that should be stored in the sqlite_stat2 table. If so,
         ** store it.  */
@@ -73473,12 +73819,17 @@ static void analyzeOneTable(
 
         sqlite3VdbeJumpHere(v, ne);
         sqlite3VdbeAddOp2(v, OP_AddImm, regRecno, 1);
-      }
 #endif
 
-      sqlite3VdbeAddOp3(v, OP_Ne, regCol, 0, iMem+nCol+i+1);
-      /**** TODO:  add collating sequence *****/
-      sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL);
+        /* Always record the very first row */
+        sqlite3VdbeAddOp1(v, OP_IfNot, iMem+1);
+      }
+      assert( pIdx->azColl!=0 );
+      assert( pIdx->azColl[i]!=0 );
+      pColl = sqlite3LocateCollSeq(pParse, pIdx->azColl[i]);
+      sqlite3VdbeAddOp4(v, OP_Ne, regCol, 0, iMem+nCol+i+1,
+                       (char*)pColl, P4_COLLSEQ);
+      sqlite3VdbeChangeP5(v, SQLITE_NULLEQ);
     }
     if( db->mallocFailed ){
       /* If a malloc failure has occurred, then the result of the expression 
@@ -73489,7 +73840,11 @@ static void analyzeOneTable(
     }
     sqlite3VdbeAddOp2(v, OP_Goto, 0, endOfLoop);
     for(i=0; i<nCol; i++){
-      sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-(nCol*2));
+      int addr2 = sqlite3VdbeCurrentAddr(v) - (nCol*2);
+      if( i==0 ){
+        sqlite3VdbeJumpHere(v, addr2-1);  /* Set jump dest for the OP_IfNot */
+      }
+      sqlite3VdbeJumpHere(v, addr2);      /* Set jump dest for the OP_Ne */
       sqlite3VdbeAddOp2(v, OP_AddImm, iMem+i+1, 1);
       sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, iMem+nCol+i+1);
     }
@@ -73835,8 +74190,11 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){
 
     if( rc==SQLITE_OK ){
       while( sqlite3_step(pStmt)==SQLITE_ROW ){
-        char *zIndex = (char *)sqlite3_column_text(pStmt, 0);
-        Index *pIdx = sqlite3FindIndex(db, zIndex, sInfo.zDatabase);
+        char *zIndex;   /* Index name */
+        Index *pIdx;    /* Pointer to the index object */
+
+        zIndex = (char *)sqlite3_column_text(pStmt, 0);
+        pIdx = zIndex ? sqlite3FindIndex(db, zIndex, sInfo.zDatabase) : 0;
         if( pIdx ){
           int iSample = sqlite3_column_int(pStmt, 1);
           if( iSample<SQLITE_INDEX_SAMPLES && iSample>=0 ){
@@ -80923,9 +81281,9 @@ SQLITE_PRIVATE void sqlite3RegisterLikeFunctions(sqlite3 *db, int caseSensitive)
   }else{
     pInfo = (struct compareInfo*)&likeInfoNorm;
   }
-  sqlite3CreateFunc(db, "like", 2, SQLITE_ANY, pInfo, likeFunc, 0, 0, 0);
-  sqlite3CreateFunc(db, "like", 3, SQLITE_ANY, pInfo, likeFunc, 0, 0, 0);
-  sqlite3CreateFunc(db, "glob", 2, SQLITE_ANY, 
+  sqlite3CreateFunc(db, "like", 2, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0);
+  sqlite3CreateFunc(db, "like", 3, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0);
+  sqlite3CreateFunc(db, "glob", 2, SQLITE_UTF8, 
       (struct compareInfo*)&globInfo, likeFunc, 0, 0, 0);
   setLikeOptFlag(db, "glob", SQLITE_FUNC_LIKE | SQLITE_FUNC_CASE);
   setLikeOptFlag(db, "like", 
@@ -87604,7 +87962,7 @@ SQLITE_API int sqlite3_prepare_v2(
 */
 static int sqlite3Prepare16(
   sqlite3 *db,              /* Database handle. */ 
-  const void *zSql,         /* UTF-8 encoded SQL statement. */
+  const void *zSql,         /* UTF-16 encoded SQL statement. */
   int nBytes,               /* Length of zSql in bytes. */
   int saveSqlFlag,          /* True to save SQL text into the sqlite3_stmt */
   sqlite3_stmt **ppStmt,    /* OUT: A pointer to the prepared statement */
@@ -87654,7 +88012,7 @@ static int sqlite3Prepare16(
 */
 SQLITE_API int sqlite3_prepare16(
   sqlite3 *db,              /* Database handle. */ 
-  const void *zSql,         /* UTF-8 encoded SQL statement. */
+  const void *zSql,         /* UTF-16 encoded SQL statement. */
   int nBytes,               /* Length of zSql in bytes. */
   sqlite3_stmt **ppStmt,    /* OUT: A pointer to the prepared statement */
   const void **pzTail       /* OUT: End of parsed string */
@@ -87666,7 +88024,7 @@ SQLITE_API int sqlite3_prepare16(
 }
 SQLITE_API int sqlite3_prepare16_v2(
   sqlite3 *db,              /* Database handle. */ 
-  const void *zSql,         /* UTF-8 encoded SQL statement. */
+  const void *zSql,         /* UTF-16 encoded SQL statement. */
   int nBytes,               /* Length of zSql in bytes. */
   sqlite3_stmt **ppStmt,    /* OUT: A pointer to the prepared statement */
   const void **pzTail       /* OUT: End of parsed string */
@@ -97675,10 +98033,9 @@ static int valueFromExpr(
   u8 aff, 
   sqlite3_value **pp
 ){
-  /* The evalConstExpr() function will have already converted any TK_VARIABLE
-  ** expression involved in an comparison into a TK_REGISTER. */
-  assert( pExpr->op!=TK_VARIABLE );
-  if( pExpr->op==TK_REGISTER && pExpr->op2==TK_VARIABLE ){
+  if( pExpr->op==TK_VARIABLE
+   || (pExpr->op==TK_REGISTER && pExpr->op2==TK_VARIABLE)
+  ){
     int iVar = pExpr->iColumn;
     sqlite3VdbeSetVarmask(pParse->pVdbe, iVar); /* IMP: R-23257-02778 */
     *pp = sqlite3VdbeGetValue(pParse->pReprepare, iVar, aff);
@@ -105466,7 +105823,7 @@ SQLITE_PRIVATE const char *sqlite3ErrStr(int rc){
     /* SQLITE_INTERRUPT   */ "interrupted",
     /* SQLITE_IOERR       */ "disk I/O error",
     /* SQLITE_CORRUPT     */ "database disk image is malformed",
-    /* SQLITE_NOTFOUND    */ 0,
+    /* SQLITE_NOTFOUND    */ "unknown operation",
     /* SQLITE_FULL        */ "database or disk is full",
     /* SQLITE_CANTOPEN    */ "unable to open database file",
     /* SQLITE_PROTOCOL    */ "locking protocol",
@@ -106471,6 +106828,9 @@ static int openDatabase(
 #endif
 #if SQLITE_DEFAULT_RECURSIVE_TRIGGERS
                  | SQLITE_RecTriggers
+#endif
+#if defined(SQLITE_DEFAULT_FOREIGN_KEYS) && SQLITE_DEFAULT_FOREIGN_KEYS
+                 | SQLITE_ForeignKeys
 #endif
       ;
   sqlite3HashInit(&db->aCollSeq);
@@ -107011,6 +107371,8 @@ SQLITE_API int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, vo
         rc = SQLITE_OK;
       }else if( fd->pMethods ){
         rc = sqlite3OsFileControl(fd, op, pArg);
+      }else{
+        rc = SQLITE_NOTFOUND;
       }
       sqlite3BtreeLeave(pBtree);
     }
@@ -111309,9 +111671,16 @@ SQLITE_PRIVATE char *sqlite3Fts3FindPositions(
   assert( pExpr->isLoaded );
   if( pExpr->aDoclist ){
     char *pEnd = &pExpr->aDoclist[pExpr->nDoclist];
-    char *pCsr = pExpr->pCurrent;
+    char *pCsr;
 
+    if( pExpr->pCurrent==0 ){
+      pExpr->pCurrent = pExpr->aDoclist;
+      pExpr->iCurrent = 0;
+      pExpr->pCurrent += sqlite3Fts3GetVarint(pExpr->pCurrent,&pExpr->iCurrent);
+    }
+    pCsr = pExpr->pCurrent;
     assert( pCsr );
+
     while( pCsr<pEnd ){
       if( pExpr->iCurrent<iDocid ){
         fts3PoslistCopy(0, &pCsr);
@@ -115500,24 +115869,28 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderCost(
       ** to right.
       */
       sqlite3_stmt *pStmt;
-      rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0);
+      sqlite3_int64 nDoc = 0;
+      sqlite3_int64 nByte = 0;
+      const char *a;
+      rc = sqlite3Fts3SelectDoctotal(p, &pStmt);
       if( rc ) return rc;
-      if( sqlite3_step(pStmt)==SQLITE_ROW ){
-        sqlite3_int64 nDoc = 0;
-        sqlite3_int64 nByte = 0;
-        const char *a = sqlite3_column_blob(pStmt, 0);
-        if( a ){
-          const char *pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
-          a += sqlite3Fts3GetVarint(a, &nDoc);
-          while( a<pEnd ){
-            a += sqlite3Fts3GetVarint(a, &nByte);
-          }
+      a = sqlite3_column_blob(pStmt, 0);
+      if( a ){
+        const char *pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
+        a += sqlite3Fts3GetVarint(a, &nDoc);
+        while( a<pEnd ){
+          a += sqlite3Fts3GetVarint(a, &nByte);
         }
-
-        pCsr->nRowAvg = (int)(((nByte / nDoc) + pgsz - 1) / pgsz);
       }
+      if( nDoc==0 || nByte==0 ){
+        sqlite3_reset(pStmt);
+        return SQLITE_CORRUPT;
+      }
+
+      pCsr->nRowAvg = (int)(((nByte / nDoc) + pgsz) / pgsz);
+      assert( pCsr->nRowAvg>0 ); 
       rc = sqlite3_reset(pStmt);
-      if( rc!=SQLITE_OK || pCsr->nRowAvg==0 ) return rc;
+      if( rc!=SQLITE_OK ) return rc;
     }
 
     /* Assume that a blob flows over onto overflow pages if it is larger
@@ -116638,7 +117011,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderIterate(
           nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0);
           if( nDoclist+nByte>nAlloc ){
             char *aNew;
-            nAlloc = nDoclist+nByte*2;
+            nAlloc = (nDoclist+nByte)*2;
             aNew = sqlite3_realloc(aBuffer, nAlloc);
             if( !aNew ){
               rc = SQLITE_NOMEM;
@@ -117515,7 +117888,7 @@ static int fts3ExprNearTrim(Fts3Expr *pExpr){
 ** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
 ** fts3ExprLoadDoclists().
 */
-static int fts3ExprLoadDoclistsCb1(Fts3Expr *pExpr, int iPhrase, void *ctx){
+static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
   int rc = SQLITE_OK;
   LoadDoclistCtx *p = (LoadDoclistCtx *)ctx;
 
@@ -117535,22 +117908,6 @@ static int fts3ExprLoadDoclistsCb1(Fts3Expr *pExpr, int iPhrase, void *ctx){
   return rc;
 }
 
-/*
-** This is an fts3ExprIterate() callback used while loading the doclists
-** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
-** fts3ExprLoadDoclists().
-*/
-static int fts3ExprLoadDoclistsCb2(Fts3Expr *pExpr, int iPhrase, void *ctx){
-  UNUSED_PARAMETER(iPhrase);
-  UNUSED_PARAMETER(ctx);
-  if( pExpr->aDoclist ){
-    pExpr->pCurrent = pExpr->aDoclist;
-    pExpr->iCurrent = 0;
-    pExpr->pCurrent += sqlite3Fts3GetVarint(pExpr->pCurrent, &pExpr->iCurrent);
-  }
-  return SQLITE_OK;
-}
-
 /*
 ** Load the doclists for each phrase in the query associated with FTS3 cursor
 ** pCsr. 
@@ -117569,10 +117926,7 @@ static int fts3ExprLoadDoclists(
   int rc;                         /* Return Code */
   LoadDoclistCtx sCtx = {0,0,0};  /* Context for fts3ExprIterate() */
   sCtx.pCsr = pCsr;
-  rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb1, (void *)&sCtx);
-  if( rc==SQLITE_OK ){
-    (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb2, 0);
-  }
+  rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb, (void *)&sCtx);
   if( pnPhrase ) *pnPhrase = sCtx.nPhrase;
   if( pnToken ) *pnToken = sCtx.nToken;
   return rc;
@@ -118264,6 +118618,7 @@ static int fts3MatchinfoSelectDoctotal(
     if( rc!=SQLITE_OK ) return rc;
   }
   pStmt = *ppStmt;
+  assert( sqlite3_data_count(pStmt)==1 );
 
   a = sqlite3_column_blob(pStmt, 0);
   a += sqlite3Fts3GetVarint(a, &nDoc);